<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />

  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  <title>Installation &mdash; nanobench  documentation</title>
      <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
      <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
    <link rel="shortcut icon" href="_static/favicon.ico"/>
  <!--[if lt IE 9]>
    <script src="_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
        <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
        <script src="_static/jquery.js"></script>
        <script src="_static/underscore.js"></script>
        <script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
        <script src="_static/doctools.js"></script>
        <script src="_static/sphinx_highlight.js"></script>
        <script async="async" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
    <script src="_static/js/theme.js"></script>
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="next" title="Test Set" href="comparison.html" />
    <link rel="prev" title="nanobench" href="index.html" /> 
</head>

<body class="wy-body-for-nav"> 
  <div class="wy-grid-for-nav">
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
            <a href="index.html" class="icon icon-home"> nanobench
            <img src="_static/nanobench-logo.svg" class="logo" alt="Logo"/>
          </a>
              <div class="version">
                v4.3.9
              </div>
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>
        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="current reference internal" href="#">Installation</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#direct-inclusion">Direct Inclusion</a></li>
<li class="toctree-l2"><a class="reference internal" href="#cmake-integration">CMake Integration</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="#usage">Usage</a></li>
<li class="toctree-l1"><a class="reference internal" href="#examples">Examples</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#something-fast">Something Fast</a></li>
<li class="toctree-l2"><a class="reference internal" href="#something-slow">Something Slow</a></li>
<li class="toctree-l2"><a class="reference internal" href="#something-unstable">Something Unstable</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="#comparing-results">Comparing Results</a></li>
<li class="toctree-l1"><a class="reference internal" href="#asymptotic-complexity">Asymptotic Complexity</a></li>
<li class="toctree-l1"><a class="reference internal" href="#rendering-mustache-like-templates">Rendering Mustache-like Templates</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#csv-comma-separated-values">CSV - Comma-Separated Values</a></li>
<li class="toctree-l2"><a class="reference internal" href="#html-box-plots">HTML Box Plots</a></li>
<li class="toctree-l2"><a class="reference internal" href="#json-javascript-object-notation">JSON - JavaScript Object Notation</a></li>
<li class="toctree-l2"><a class="reference internal" href="#pyperf-python-pyperf-module-output">pyperf - Python pyperf module Output</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#show-benchmark-statistics">Show Benchmark Statistics</a></li>
<li class="toctree-l3"><a class="reference internal" href="#show-a-histogram">Show a Histogram</a></li>
<li class="toctree-l3"><a class="reference internal" href="#compare-results">Compare Results</a></li>
</ul>
</li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Comparison</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="comparison.html">Test Set</a></li>
<li class="toctree-l1"><a class="reference internal" href="comparison.html#runtime">Runtime</a></li>
<li class="toctree-l1"><a class="reference internal" href="comparison.html#implementations-output">Implementations &amp; Output</a><ul>
<li class="toctree-l2"><a class="reference internal" href="comparison.html#nanobench">nanobench</a><ul>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#sourcecode">Sourcecode</a></li>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#results">Results</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="comparison.html#google-benchmark">Google Benchmark</a><ul>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id2">Sourcecode</a></li>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id3">Results</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="comparison.html#nonius">nonius</a><ul>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id5">Sourcecode</a></li>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id6">Results</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="comparison.html#picobench">Picobench</a><ul>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id8">Sourcecode</a></li>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id9">Results</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="comparison.html#catch2">Catch2</a><ul>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id11">Sourcecode</a></li>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id12">Results</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="comparison.html#moodycamel-microbench">moodycamel::microbench</a><ul>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id14">Sourcecode</a></li>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id15">Results</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="comparison.html#sltbench">sltbench</a><ul>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id17">Sourcecode</a></li>
<li class="toctree-l3"><a class="reference internal" href="comparison.html#id18">Results</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="comparison.html#celero">Celero</a></li>
<li class="toctree-l2"><a class="reference internal" href="comparison.html#folly-benchmark">folly Benchmark</a></li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Reference</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="reference.html"><code class="docutils literal notranslate"><span class="pre">ankerl::nanobench</span></code> Reference</a><ul>
<li class="toctree-l2"><a class="reference internal" href="reference.html#bench-main-entry-point"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">Bench</span></code> - Main Entry Point</a></li>
<li class="toctree-l2"><a class="reference internal" href="reference.html#rng-extremely-fast-prng"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">Rng</span></code> - Extremely fast PRNG</a></li>
<li class="toctree-l2"><a class="reference internal" href="reference.html#result-benchmark-results"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">Result</span></code> - Benchmark Results</a></li>
<li class="toctree-l2"><a class="reference internal" href="reference.html#donotoptimizeaway"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">doNotOptimizeAway()</span></code></a></li>
<li class="toctree-l2"><a class="reference internal" href="reference.html#render-mustache-like-templates"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">render()</span></code> - Mustache-like Templates</a><ul>
<li class="toctree-l3"><a class="reference internal" href="reference.html#templates-csv"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">templates::csv</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="reference.html#templates-htmlboxplot"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">templates::htmlBoxplot</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="reference.html#templates-json"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">templates::json</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="reference.html#templates-pyperf"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">templates::pyperf</span></code></a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="reference.html#environment-variables">Environment Variables</a><ul>
<li class="toctree-l3"><a class="reference internal" href="reference.html#nanobench-endless-run-a-specific-test-endlessly"><code class="docutils literal notranslate"><span class="pre">NANOBENCH_ENDLESS</span></code> - Run a Specific Test Endlessly</a></li>
<li class="toctree-l3"><a class="reference internal" href="reference.html#nanobench-suppress-warnings-no-stability-warnings"><code class="docutils literal notranslate"><span class="pre">NANOBENCH_SUPPRESS_WARNINGS</span></code> - No Stability Warnings</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="genindex.html">Index</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">About</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="license.html">MIT License</a></li>
<li class="toctree-l1"><a class="reference internal" href="CODE_OF_CONDUCT.html">Contributor Covenant Code of Conduct</a><ul>
<li class="toctree-l2"><a class="reference internal" href="CODE_OF_CONDUCT.html#our-pledge">Our Pledge</a></li>
<li class="toctree-l2"><a class="reference internal" href="CODE_OF_CONDUCT.html#our-standards">Our Standards</a></li>
<li class="toctree-l2"><a class="reference internal" href="CODE_OF_CONDUCT.html#enforcement-responsibilities">Enforcement Responsibilities</a></li>
<li class="toctree-l2"><a class="reference internal" href="CODE_OF_CONDUCT.html#scope">Scope</a></li>
<li class="toctree-l2"><a class="reference internal" href="CODE_OF_CONDUCT.html#enforcement">Enforcement</a></li>
<li class="toctree-l2"><a class="reference internal" href="CODE_OF_CONDUCT.html#enforcement-guidelines">Enforcement Guidelines</a><ul>
<li class="toctree-l3"><a class="reference internal" href="CODE_OF_CONDUCT.html#correction">1. Correction</a></li>
<li class="toctree-l3"><a class="reference internal" href="CODE_OF_CONDUCT.html#warning">2. Warning</a></li>
<li class="toctree-l3"><a class="reference internal" href="CODE_OF_CONDUCT.html#temporary-ban">3. Temporary Ban</a></li>
<li class="toctree-l3"><a class="reference internal" href="CODE_OF_CONDUCT.html#permanent-ban">4. Permanent Ban</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="CODE_OF_CONDUCT.html#attribution">Attribution</a></li>
</ul>
</li>
</ul>

        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="index.html">nanobench</a>
      </nav>

      <div class="wy-nav-content">
        <div class="rst-content">
          <div role="navigation" aria-label="Page navigation">
  <ul class="wy-breadcrumbs">
      <li><a href="index.html" class="icon icon-home"></a> &raquo;</li>
      <li>Installation</li>
      <li class="wy-breadcrumbs-aside">
              <a href="https://github.com/martinus/nanobench/blob/master/src/docs/tutorial.rst" class="fa fa-github"> Edit on GitHub</a>
      </li>
  </ul>
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
             
  <section id="installation">
<h1>Installation<a class="headerlink" href="#installation" title="Permalink to this heading"></a></h1>
<section id="direct-inclusion">
<h2>Direct Inclusion<a class="headerlink" href="#direct-inclusion" title="Permalink to this heading"></a></h2>
<ol class="arabic">
<li><p>Download <code class="docutils literal notranslate"><span class="pre">nanobench.h</span></code> from the <a class="reference download external" download="" href="https://github.com/martinus/nanobench/releases/latest"><code class="xref download docutils literal notranslate"><span class="pre">release</span></code></a>
and make it available in your project.</p></li>
<li><p>Create a .cpp file, e.g. <code class="docutils literal notranslate"><span class="pre">nanobench.cpp</span></code>, where the bulk of nanobench is compiled.</p>
<div class="literal-block-wrapper docutils container" id="id2">
<div class="code-block-caption"><span class="caption-text">nanobench.cpp</span><a class="headerlink" href="#id2" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos">1</span><span class="cp">#define ANKERL_NANOBENCH_IMPLEMENT</span>
<span class="linenos">2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
</pre></div>
</div>
</div>
</li>
<li><p>Compile e.g. with <code class="docutils literal notranslate"><span class="pre">g++</span> <span class="pre">-O3</span> <span class="pre">-I../include</span> <span class="pre">-c</span> <span class="pre">nanobench.cpp</span></code>. This compiles
the bulk of nanobench, and took 2.4 seconds on my machine. It needs to be compiled only once whenever you upgrade nanobench.</p></li>
</ol>
</section>
<section id="cmake-integration">
<h2>CMake Integration<a class="headerlink" href="#cmake-integration" title="Permalink to this heading"></a></h2>
<p><code class="docutils literal notranslate"><span class="pre">nanobench</span></code> can be integrated with CMake’s <a class="reference external" href="https://cmake.org/cmake/help/latest/module/FetchContent.html">FetchContent</a> or as
a <a class="reference external" href="https://git-scm.com/book/en/v2/Git-Tools-Submodules">git submodule</a>. Here is a full example how to this can be done:</p>
<div class="literal-block-wrapper docutils container" id="id3">
<div class="code-block-caption"><span class="caption-text">CMakeLists.txt</span><a class="headerlink" href="#id3" title="Permalink to this code"></a></div>
<div class="highlight-CMake notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="nb">cmake_minimum_required</span><span class="p">(</span><span class="s">VERSION</span><span class="w"> </span><span class="s">3.14</span><span class="p">)</span>
<span class="linenos"> 2</span><span class="nb">set</span><span class="p">(</span><span class="s">CMAKE_CXX_STANDARD</span><span class="w"> </span><span class="s">17</span><span class="p">)</span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="nb">project</span><span class="p">(</span>
<span class="linenos"> 5</span><span class="w">    </span><span class="s">CMakeNanobenchExample</span>
<span class="linenos"> 6</span><span class="w">    </span><span class="s">VERSION</span><span class="w"> </span><span class="s">1.0</span>
<span class="linenos"> 7</span><span class="w">    </span><span class="s">LANGUAGES</span><span class="w"> </span><span class="s">CXX</span><span class="p">)</span>
<span class="linenos"> 8</span>
<span class="linenos"> 9</span><span class="nb">include</span><span class="p">(</span><span class="s">FetchContent</span><span class="p">)</span>
<span class="linenos">10</span>
<span class="linenos">11</span><span class="nb">FetchContent_Declare</span><span class="p">(</span>
<span class="linenos">12</span><span class="w">    </span><span class="s">nanobench</span>
<span class="linenos">13</span><span class="w">    </span><span class="s">GIT_REPOSITORY</span><span class="w"> </span><span class="s">https://github.com/martinus/nanobench.git</span>
<span class="linenos">14</span><span class="w">    </span><span class="s">GIT_TAG</span><span class="w"> </span><span class="s">v4.1.0</span>
<span class="linenos">15</span><span class="w">    </span><span class="s">GIT_SHALLOW</span><span class="w"> </span><span class="s">TRUE</span><span class="p">)</span>
<span class="linenos">16</span>
<span class="linenos">17</span><span class="nb">FetchContent_MakeAvailable</span><span class="p">(</span><span class="s">nanobench</span><span class="p">)</span>
<span class="linenos">18</span>
<span class="linenos">19</span><span class="nb">add_executable</span><span class="p">(</span><span class="s">MyExample</span><span class="w"> </span><span class="s">my_example.cpp</span><span class="p">)</span>
<span class="linenos">20</span><span class="nb">target_link_libraries</span><span class="p">(</span><span class="s">MyExample</span><span class="w"> </span><span class="s">PRIVATE</span><span class="w"> </span><span class="s">nanobench</span><span class="p">)</span>
</pre></div>
</div>
</div>
</section>
</section>
<section id="usage">
<h1>Usage<a class="headerlink" href="#usage" title="Permalink to this heading"></a></h1>
<ol class="arabic">
<li><p>Create the actual benchmark code, in <code class="docutils literal notranslate"><span class="pre">full_example.cpp</span></code>:</p>
<div class="literal-block-wrapper docutils container" id="id4">
<div class="code-block-caption"><span class="caption-text">full_example.cpp</span><a class="headerlink" href="#id4" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span>
<span class="linenos"> 3</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;atomic&gt;</span><span class="cp"></span>
<span class="linenos"> 4</span>
<span class="linenos"> 5</span><span class="kt">int</span><span class="w"> </span><span class="nf">main</span><span class="p">()</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 6</span><span class="w">    </span><span class="kt">int</span><span class="w"> </span><span class="n">y</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"></span>
<span class="linenos"> 7</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">atomic</span><span class="o">&lt;</span><span class="kt">int</span><span class="o">&gt;</span><span class="w"> </span><span class="n">x</span><span class="p">(</span><span class="mi">0</span><span class="p">);</span><span class="w"></span>
<span class="linenos"> 8</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">().</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;compare_exchange_strong&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 9</span><span class="w">        </span><span class="n">x</span><span class="p">.</span><span class="n">compare_exchange_strong</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="w"> </span><span class="mi">0</span><span class="p">);</span><span class="w"></span>
<span class="linenos">10</span><span class="w">    </span><span class="p">});</span><span class="w"></span>
<span class="linenos">11</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>The most important entry entry point is <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench5BenchE" title="ankerl::nanobench::Bench"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">ankerl::nanobench::Bench</span></code></a>. It creates a benchmarking object,
optionally configures it, and then runs the code to benchmark with <a class="reference internal" href="reference.html#_CPPv4I0EN6ankerl9nanobench5Bench3runER5BenchPKcRR2Op" title="ankerl::nanobench::Bench::run"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">run()</span></code></a>.</p>
</li>
<li><p>Compile and link the example with</p>
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span>g++ -O3 -I../include nanobench.o full_example.cpp -o full_example
</pre></div>
</div>
<p>This takes just 0.28 seconds on my machine.</p>
</li>
<li><p>Run <code class="docutils literal notranslate"><span class="pre">./full_example</span></code>, which gives an output like this:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>|               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
|                5.63 |      177,595,338.98 |    0.0% |            3.00 |           17.98 |  0.167 |           1.00 |    0.1% |      0.00 | `compare_exchange_strong`
</pre></div>
</div>
<p>Which renders as</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 13%" />
<col style="width: 13%" />
<col style="width: 6%" />
<col style="width: 11%" />
<col style="width: 11%" />
<col style="width: 5%" />
<col style="width: 10%" />
<col style="width: 6%" />
<col style="width: 7%" />
<col style="width: 18%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>ns/op</p></th>
<th class="head"><p>op/s</p></th>
<th class="head"><p>err%</p></th>
<th class="head"><p>ins/op</p></th>
<th class="head"><p>cyc/op</p></th>
<th class="head"><p>IPC</p></th>
<th class="head"><p>bra/op</p></th>
<th class="head"><p>miss%</p></th>
<th class="head"><p>total</p></th>
<th class="head"><p>benchmark</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>5.63</p></td>
<td><p>177,595,338.98</p></td>
<td><p>0.0%</p></td>
<td><p>3.00</p></td>
<td><p>17.98</p></td>
<td><p>0.167</p></td>
<td><p>1.00</p></td>
<td><p>0.1%</p></td>
<td><p>0.00</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">compare_exchange_strong</span></code></p></td>
</tr>
</tbody>
</table>
<p>Which means that one <code class="docutils literal notranslate"><span class="pre">x.compare_exchange_strong(y,</span> <span class="pre">0);</span></code> call takes
5.63ns on my machine (wall-clock time), or ~178 million operations per second. Runtime
fluctuates by around 0.0%, so the results are very stable. Each call
required 3 instructions, which took ~18 CPU cycles. There was a single branch per call,
with only 0.1% mispredicted.</p>
</li>
</ol>
<p>Nanobench does not come with a test runner, so you can easily use it with any framework you like.  In the remaining examples, I’m
using <a class="reference external" href="https://github.com/onqtam/doctest">doctest</a> as a unit test framework.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>CPU statistics like instructions, cycles, branches, branch misses are only available on Linux, through
<a class="reference external" href="http://web.eece.maine.edu/~vweaver/projects/perf_events/">perf events</a>. On some systems you might need to
<a class="reference external" href="https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html#unprivileged-users">change permissions</a>
through <code class="docutils literal notranslate"><span class="pre">perf_event_paranoid</span></code> or use ACL.</p>
</div>
</section>
<section id="examples">
<h1>Examples<a class="headerlink" href="#examples" title="Permalink to this heading"></a></h1>
<section id="something-fast">
<h2>Something Fast<a class="headerlink" href="#something-fast" title="Permalink to this heading"></a></h2>
<p>Let’s benchmarks how fast we can do <code class="docutils literal notranslate"><span class="pre">x</span> <span class="pre">+=</span> <span class="pre">x</span></code> for <code class="docutils literal notranslate"><span class="pre">uint64_t</span></code>:</p>
<div class="literal-block-wrapper docutils container" id="id5">
<div class="code-block-caption"><span class="caption-text">tutorial_fast_v1.cpp</span><a class="headerlink" href="#id5" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos">1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos">2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos">3</span>
<span class="linenos">4</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_fast_v1&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">5</span><span class="w">    </span><span class="kt">uint64_t</span><span class="w"> </span><span class="n">x</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span><span class="w"></span>
<span class="linenos">6</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">().</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;++x&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]()</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">7</span><span class="w">        </span><span class="o">++</span><span class="n">x</span><span class="p">;</span><span class="w"></span>
<span class="linenos">8</span><span class="w">    </span><span class="p">});</span><span class="w"></span>
<span class="linenos">9</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>After 0.2ms we get this output:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>|               ns/op |                op/s |    err% |     total | benchmark
|--------------------:|--------------------:|--------:|----------:|:----------
|                   - |                   - |       - |         - | :boom: `++x` (iterations overflow. Maybe your code got optimized away?)
</pre></div>
</div>
<p>No data there! We only get <code class="docutils literal notranslate"><span class="pre">:boom:</span> <span class="pre">iterations</span> <span class="pre">overflow.</span></code>.  The compiler could optimize <code class="docutils literal notranslate"><span class="pre">x</span> <span class="pre">+=</span> <span class="pre">x</span></code>
away because we never used the output. Thanks to <code class="docutils literal notranslate"><span class="pre">doNotOptimizeAway</span></code>, this is easy to fix:</p>
<div class="literal-block-wrapper docutils container" id="id6">
<div class="code-block-caption"><span class="caption-text">tutorial_fast_v2.cpp</span><a class="headerlink" href="#id6" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos">1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos">2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos">3</span>
<span class="linenos">4</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_fast_v2&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">5</span><span class="w">    </span><span class="kt">uint64_t</span><span class="w"> </span><span class="n">x</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span><span class="w"></span>
<span class="linenos">6</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">().</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;++x&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]()</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="hll"><span class="linenos">7</span><span class="w">        </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">doNotOptimizeAway</span><span class="p">(</span><span class="n">x</span><span class="w"> </span><span class="o">+=</span><span class="w"> </span><span class="mi">1</span><span class="p">);</span><span class="w"></span>
</span><span class="linenos">8</span><span class="w">    </span><span class="p">});</span><span class="w"></span>
<span class="linenos">9</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>This time the benchmark runs for 2.2ms and we actually get reasonable data:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>|               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
|                0.31 |    3,192,444,232.50 |    0.0% |            1.00 |            1.00 |  0.998 |           0.00 |    0.0% |      0.00 | `++x`
</pre></div>
</div>
<p>It’s a very stable result. One run the op/s is 3,192 million/sec, the next time I execute it I get 3,168 million/sec. It always takes
1.00 instructions per operation on my machine, and can do this in ~1 cycle.</p>
</section>
<section id="something-slow">
<h2>Something Slow<a class="headerlink" href="#something-slow" title="Permalink to this heading"></a></h2>
<p>Let’s benchmark if sleeping for 100ms really takes 100ms.</p>
<div class="literal-block-wrapper docutils container" id="id7">
<div class="code-block-caption"><span class="caption-text">tutorial_slow_v1.cpp</span><a class="headerlink" href="#id7" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;chrono&gt;</span><span class="cp"></span>
<span class="linenos"> 5</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thread&gt;</span><span class="cp"></span>
<span class="linenos"> 6</span>
<span class="linenos"> 7</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_slow_v1&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 8</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">().</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;sleep 100ms, auto&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 9</span><span class="w">        </span><span class="n">std</span><span class="o">::</span><span class="n">this_thread</span><span class="o">::</span><span class="n">sleep_for</span><span class="p">(</span><span class="n">std</span><span class="o">::</span><span class="n">chrono</span><span class="o">::</span><span class="n">milliseconds</span><span class="p">(</span><span class="mi">100</span><span class="p">));</span><span class="w"></span>
<span class="linenos">10</span><span class="w">    </span><span class="p">});</span><span class="w"></span>
<span class="linenos">11</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>After 1.1 seconds I get</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>|               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------------------
|      100,125,753.00 |                9.99 |    0.0% |           51.00 |        7,714.00 |  0.007 |          11.00 |   90.9% |      1.10 | `sleep 100ms, auto`
</pre></div>
</div>
<p>So we actually take 100.125ms instead of 100ms. Next time I run it, I get 100.141. Also a very stable result. Interestingly, sleep takes 51 instructions but 7,714 cycles - so we only got 0.007 instructions per cycle. That’s extremely low, but expected of <code class="docutils literal notranslate"><span class="pre">sleep</span></code>. It also required 11 branches, of which 90.9% were mispredicted on average.</p>
<p>If the extremely slow 1.1 second is too much for you, you can manually configure the number of evaluations (epochs):</p>
<div class="literal-block-wrapper docutils container" id="id8">
<div class="code-block-caption"><span class="caption-text">tutorial_slow_v2.cpp</span><a class="headerlink" href="#id8" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;chrono&gt;</span><span class="cp"></span>
<span class="linenos"> 5</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thread&gt;</span><span class="cp"></span>
<span class="linenos"> 6</span>
<span class="linenos"> 7</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_slow_v2&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 8</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">().</span><span class="n">epochs</span><span class="p">(</span><span class="mi">3</span><span class="p">).</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;sleep 100ms&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 9</span><span class="w">        </span><span class="n">std</span><span class="o">::</span><span class="n">this_thread</span><span class="o">::</span><span class="n">sleep_for</span><span class="p">(</span><span class="n">std</span><span class="o">::</span><span class="n">chrono</span><span class="o">::</span><span class="n">milliseconds</span><span class="p">(</span><span class="mi">100</span><span class="p">));</span><span class="w"></span>
<span class="linenos">10</span><span class="w">    </span><span class="p">});</span><span class="w"></span>
<span class="linenos">11</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>|               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
|      100,099,096.00 |                9.99 |    0.0% |           51.00 |        7,182.00 |  0.007 |          11.00 |   90.9% |      0.30 | `sleep 100ms`
</pre></div>
</div>
<p>This time it took only 0.3 seconds, but with only 3 evaluations instead of 11. The err% will be less meaningful, but since the benchmark is so stable it doesn’t really matter.</p>
</section>
<section id="something-unstable">
<h2>Something Unstable<a class="headerlink" href="#something-unstable" title="Permalink to this heading"></a></h2>
<p>Let’s create an extreme artificial test that’s hard to benchmark, because runtime fluctuates randomly: Each iteration
randomly skip between 0-254 random numbers:</p>
<div class="literal-block-wrapper docutils container" id="id9">
<div class="code-block-caption"><span class="caption-text">tutorial_fluctuating_v1.cpp</span><a class="headerlink" href="#id9" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;random&gt;</span><span class="cp"></span>
<span class="linenos"> 5</span>
<span class="linenos"> 6</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_fluctuating_v1&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 7</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">random_device</span><span class="w"> </span><span class="n">dev</span><span class="p">;</span><span class="w"></span>
<span class="linenos"> 8</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">mt19937_64</span><span class="w"> </span><span class="nf">rng</span><span class="p">(</span><span class="n">dev</span><span class="p">());</span><span class="w"></span>
<span class="linenos"> 9</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">().</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;random fluctuations&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">10</span><span class="w">        </span><span class="c1">// each run, perform a random number of rng calls</span>
<span class="linenos">11</span><span class="w">        </span><span class="k">auto</span><span class="w"> </span><span class="n">iterations</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">rng</span><span class="p">()</span><span class="w"> </span><span class="o">&amp;</span><span class="w"> </span><span class="n">UINT64_C</span><span class="p">(</span><span class="mh">0xff</span><span class="p">);</span><span class="w"></span>
<span class="linenos">12</span><span class="w">        </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">uint64_t</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">&lt;</span><span class="w"> </span><span class="n">iterations</span><span class="p">;</span><span class="w"> </span><span class="o">++</span><span class="n">i</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">13</span><span class="w">            </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">doNotOptimizeAway</span><span class="p">(</span><span class="n">rng</span><span class="p">());</span><span class="w"></span>
<span class="linenos">14</span><span class="w">        </span><span class="p">}</span><span class="w"></span>
<span class="linenos">15</span><span class="w">    </span><span class="p">});</span><span class="w"></span>
<span class="linenos">16</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>After 2.3ms, I get this result:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>|               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
|              334.12 |        2,992,911.53 |    6.3% |        3,486.44 |        1,068.67 |  3.262 |         287.86 |    0.7% |      0.00 | :wavy_dash: `random fluctuations` (Unstable with ~56.7 iters. Increase `minEpochIterations` to e.g. 567)
</pre></div>
</div>
<p>So on average each loop takes about 334.12ns, but we get a warning that the results are unstable. The median percentage error is 6.3% which is quite high,</p>
<p>Let’s use the suggestion and set the minimum number of iterations to 5000, and try again:</p>
<div class="literal-block-wrapper docutils container" id="id10">
<div class="code-block-caption"><span class="caption-text">tutorial_fluctuating_v2.cpp</span><a class="headerlink" href="#id10" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;random&gt;</span><span class="cp"></span>
<span class="linenos"> 5</span>
<span class="linenos"> 6</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_fluctuating_v2&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 7</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">random_device</span><span class="w"> </span><span class="n">dev</span><span class="p">;</span><span class="w"></span>
<span class="linenos"> 8</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">mt19937_64</span><span class="w"> </span><span class="nf">rng</span><span class="p">(</span><span class="n">dev</span><span class="p">());</span><span class="w"></span>
<span class="hll"><span class="linenos"> 9</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">().</span><span class="n">minEpochIterations</span><span class="p">(</span><span class="mi">5000</span><span class="p">).</span><span class="n">run</span><span class="p">(</span><span class="w"></span>
</span><span class="linenos">10</span><span class="w">        </span><span class="s">&quot;random fluctuations&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">11</span><span class="w">            </span><span class="c1">// each run, perform a random number of rng calls</span>
<span class="linenos">12</span><span class="w">            </span><span class="k">auto</span><span class="w"> </span><span class="n">iterations</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">rng</span><span class="p">()</span><span class="w"> </span><span class="o">&amp;</span><span class="w"> </span><span class="n">UINT64_C</span><span class="p">(</span><span class="mh">0xff</span><span class="p">);</span><span class="w"></span>
<span class="linenos">13</span><span class="w">            </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">uint64_t</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">&lt;</span><span class="w"> </span><span class="n">iterations</span><span class="p">;</span><span class="w"> </span><span class="o">++</span><span class="n">i</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">14</span><span class="w">                </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">doNotOptimizeAway</span><span class="p">(</span><span class="n">rng</span><span class="p">());</span><span class="w"></span>
<span class="linenos">15</span><span class="w">            </span><span class="p">}</span><span class="w"></span>
<span class="linenos">16</span><span class="w">        </span><span class="p">});</span><span class="w"></span>
<span class="linenos">17</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>The fluctuations are much better:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>|               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
|              277.31 |        3,606,106.48 |    0.7% |        3,531.75 |          885.18 |  3.990 |         291.59 |    0.7% |      0.00 | `random fluctuations`
</pre></div>
</div>
<p>The results are  more stable, with only 0.7% error.</p>
</section>
</section>
<section id="comparing-results">
<span id="tutorial-comparing-results"></span><h1>Comparing Results<a class="headerlink" href="#comparing-results" title="Permalink to this heading"></a></h1>
<p>To compare results, keep the <cite>ankerl::nanobench::Bench</cite> object around, enable <cite>.relative(true)</cite>, and <cite>.run(…)</cite> your benchmarks. All benchmarks will be automatically compared to the first one.</p>
<p>As an example, I have implemented a comparison of multiple random number generators.
Here several RNGs are compared to a baseline calculated from <cite>std::default_random_engine</cite>.
I factored out the general benchmarking code so it’s easy to use for each of the random number generators:</p>
<div class="literal-block-wrapper docutils container" id="id11">
<div class="code-block-caption"><span class="caption-text">example_random_number_generators.cpp (excerpt)</span><a class="headerlink" href="#id11" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="k">private</span><span class="o">:</span><span class="w"></span>
<span class="linenos"> 2</span><span class="w">    </span><span class="k">static</span><span class="w"> </span><span class="k">constexpr</span><span class="w"> </span><span class="kt">uint64_t</span><span class="w"> </span><span class="n">rotl</span><span class="p">(</span><span class="kt">uint64_t</span><span class="w"> </span><span class="n">x</span><span class="p">,</span><span class="w"> </span><span class="kt">unsigned</span><span class="w"> </span><span class="n">k</span><span class="p">)</span><span class="w"> </span><span class="k">noexcept</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 3</span><span class="w">        </span><span class="k">return</span><span class="w"> </span><span class="p">(</span><span class="n">x</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">k</span><span class="p">)</span><span class="w"> </span><span class="o">|</span><span class="w"> </span><span class="p">(</span><span class="n">x</span><span class="w"> </span><span class="o">&gt;&gt;</span><span class="w"> </span><span class="p">(</span><span class="mi">64U</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="n">k</span><span class="p">));</span><span class="w"></span>
<span class="linenos"> 4</span><span class="w">    </span><span class="p">}</span><span class="w"></span>
<span class="linenos"> 5</span>
<span class="linenos"> 6</span><span class="w">    </span><span class="kt">uint64_t</span><span class="w"> </span><span class="n">stateA</span><span class="p">;</span><span class="w"></span>
<span class="linenos"> 7</span><span class="w">    </span><span class="kt">uint64_t</span><span class="w"> </span><span class="n">stateB</span><span class="p">;</span><span class="w"></span>
<span class="linenos"> 8</span><span class="p">};</span><span class="w"></span>
<span class="linenos"> 9</span>
<span class="linenos">10</span><span class="k">namespace</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">11</span>
<span class="linenos">12</span><span class="c1">// Benchmarks how fast we can get 64bit random values from Rng.</span>
<span class="linenos">13</span><span class="k">template</span><span class="w"> </span><span class="o">&lt;</span><span class="k">typename</span><span class="w"> </span><span class="nc">Rng</span><span class="o">&gt;</span><span class="w"></span>
<span class="linenos">14</span><span class="kt">void</span><span class="w"> </span><span class="n">bench</span><span class="p">(</span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="o">*</span><span class="w"> </span><span class="n">bench</span><span class="p">,</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="k">const</span><span class="o">*</span><span class="w"> </span><span class="n">name</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">15</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">random_device</span><span class="w"> </span><span class="n">dev</span><span class="p">;</span><span class="w"></span>
<span class="linenos">16</span><span class="w">    </span><span class="n">Rng</span><span class="w"> </span><span class="nf">rng</span><span class="p">(</span><span class="n">dev</span><span class="p">());</span><span class="w"></span>
<span class="linenos">17</span>
<span class="linenos">18</span><span class="w">    </span><span class="n">bench</span><span class="o">-&gt;</span><span class="n">run</span><span class="p">(</span><span class="n">name</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]()</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">19</span><span class="w">        </span><span class="k">auto</span><span class="w"> </span><span class="n">r</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">uniform_int_distribution</span><span class="o">&lt;</span><span class="kt">uint64_t</span><span class="o">&gt;</span><span class="p">{}(</span><span class="n">rng</span><span class="p">);</span><span class="w"></span>
<span class="linenos">20</span><span class="w">        </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">doNotOptimizeAway</span><span class="p">(</span><span class="n">r</span><span class="p">);</span><span class="w"></span>
<span class="linenos">21</span><span class="w">    </span><span class="p">});</span><span class="w"></span>
<span class="linenos">22</span><span class="p">}</span><span class="w"></span>
<span class="linenos">23</span>
<span class="linenos">24</span><span class="p">}</span><span class="w"> </span><span class="c1">// namespace</span>
<span class="linenos">25</span>
<span class="linenos">26</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;example_random_number_generators&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">27</span><span class="w">    </span><span class="c1">// perform a few warmup calls, and since the runtime is not always stable</span>
<span class="linenos">28</span><span class="w">    </span><span class="c1">// for each generator, increase the number of epochs to get more accurate</span>
<span class="linenos">29</span><span class="w">    </span><span class="c1">// numbers.</span>
<span class="linenos">30</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="w"> </span><span class="n">b</span><span class="p">;</span><span class="w"></span>
<span class="linenos">31</span><span class="w">    </span><span class="n">b</span><span class="p">.</span><span class="n">title</span><span class="p">(</span><span class="s">&quot;Random Number Generators&quot;</span><span class="p">)</span><span class="w"></span>
<span class="linenos">32</span><span class="w">        </span><span class="p">.</span><span class="n">unit</span><span class="p">(</span><span class="s">&quot;uint64_t&quot;</span><span class="p">)</span><span class="w"></span>
<span class="linenos">33</span><span class="w">        </span><span class="p">.</span><span class="n">warmup</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span><span class="w"></span>
<span class="linenos">34</span><span class="w">        </span><span class="p">.</span><span class="n">relative</span><span class="p">(</span><span class="nb">true</span><span class="p">);</span><span class="w"></span>
<span class="linenos">35</span><span class="w">    </span><span class="n">b</span><span class="p">.</span><span class="n">performanceCounters</span><span class="p">(</span><span class="nb">true</span><span class="p">);</span><span class="w"></span>
<span class="linenos">36</span>
<span class="linenos">37</span><span class="w">    </span><span class="c1">// sets the first one as the baseline</span>
<span class="linenos">38</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">default_random_engine</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;std::default_random_engine&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">39</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">mt19937</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;std::mt19937&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">40</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">mt19937_64</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;std::mt19937_64&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">41</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">ranlux24_base</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;std::ranlux24_base&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">42</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">ranlux48_base</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;std::ranlux48_base&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">43</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">ranlux24</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;std::ranlux24_base&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">44</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">ranlux48</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;std::ranlux48&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">45</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">std</span><span class="o">::</span><span class="n">knuth_b</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;std::knuth_b&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">46</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">WyRng</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;WyRng&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">47</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">NasamRng</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;NasamRng&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">48</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">Sfc4</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;Sfc4&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">49</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">RomuTrio</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;RomuTrio&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">50</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">RomuDuo</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;RomuDuo&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">51</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">RomuDuoJr</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;RomuDuoJr&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">52</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">Orbit</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;Orbit&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">53</span><span class="w">    </span><span class="n">bench</span><span class="o">&lt;</span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Rng</span><span class="o">&gt;</span><span class="p">(</span><span class="o">&amp;</span><span class="n">b</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;ankerl::nanobench::Rng&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">54</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>Runs for 60ms and prints this table:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>| relative |         ns/uint64_t |          uint64_t/s |    err% |    ins/uint64_t |    cyc/uint64_t |    IPC |   bra/uint64_t |   miss% |     total | Random Number Generators
|---------:|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:-------------------------
|   100.0% |               35.87 |       27,881,924.28 |    2.3% |          127.80 |          114.61 |  1.115 |           9.77 |    3.7% |      0.00 | `std::default_random_engine`
|   490.3% |                7.32 |      136,699,693.21 |    0.6% |           89.55 |           23.49 |  3.812 |           9.51 |    0.1% |      0.00 | `std::mt19937`
| 1,767.4% |                2.03 |      492,786,582.33 |    0.6% |           24.38 |            6.48 |  3.761 |           1.26 |    0.6% |      0.00 | `std::mt19937_64`
|    85.2% |               42.08 |       23,764,853.03 |    0.7% |          157.07 |          134.62 |  1.167 |          19.51 |    7.6% |      0.00 | `std::ranlux24_base`
|   121.3% |               29.56 |       33,824,759.51 |    0.5% |           91.03 |           94.35 |  0.965 |          10.00 |    8.1% |      0.00 | `std::ranlux48_base`
|    17.4% |              205.67 |        4,862,080.59 |    1.2% |          709.83 |          657.10 |  1.080 |         101.79 |   16.1% |      0.00 | `std::ranlux24_base`
|     8.7% |              412.46 |        2,424,497.97 |    1.8% |        1,514.70 |        1,318.43 |  1.149 |         219.09 |   16.7% |      0.00 | `std::ranlux48`
|    59.2% |               60.60 |       16,502,276.18 |    1.9% |          253.77 |          193.39 |  1.312 |          24.93 |    1.5% |      0.00 | `std::knuth_b`
| 5,187.1% |                0.69 |    1,446,254,071.66 |    0.1% |            6.00 |            2.21 |  2.714 |           0.00 |    0.0% |      0.00 | `WyRng`
| 1,431.7% |                2.51 |      399,177,833.54 |    0.0% |           21.00 |            8.01 |  2.621 |           0.00 |    0.0% |      0.00 | `NasamRng`
| 2,629.9% |                1.36 |      733,279,957.30 |    0.1% |           13.00 |            4.36 |  2.982 |           0.00 |    0.0% |      0.00 | `Sfc4`
| 3,815.7% |                0.94 |    1,063,889,655.17 |    0.0% |           11.00 |            3.01 |  3.661 |           0.00 |    0.0% |      0.00 | `RomuTrio`
| 3,529.5% |                1.02 |      984,102,081.37 |    0.3% |            9.00 |            3.25 |  2.768 |           0.00 |    0.0% |      0.00 | `RomuDuo`
| 4,580.4% |                0.78 |    1,277,113,402.06 |    0.0% |            7.00 |            2.50 |  2.797 |           0.00 |    0.0% |      0.00 | `RomuDuoJr`
| 2,291.2% |                1.57 |      638,820,992.09 |    0.0% |           11.00 |            5.00 |  2.200 |           0.00 |    0.0% |      0.00 | `ankerl::nanobench::Rng`
</pre></div>
</div>
<p>It shows that <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench3RngE" title="ankerl::nanobench::Rng"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">ankerl::nanobench::Rng</span></code></a> is one of the fastest RNG, and has the least amount of
fluctuation. It takes only 1.57ns to generate a random <code class="docutils literal notranslate"><span class="pre">uint64_t</span></code>, so ~638 million calls per
seconds are possible. To the left we show relative performance compared to <code class="docutils literal notranslate"><span class="pre">std::default_random_engine</span></code>.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Here pure runtime performance is not necessarily the best benchmark.
Especially the fastest RNG’s can be inlined and use instruction level parallelism
to their advantage: they immediately return an old state, and while user code can
already use that value, the next value is calculated in parallel. See the excellent paper
at <a class="reference external" href="http://www.romu-random.org">romu-random</a> for details.</p>
</div>
</section>
<section id="asymptotic-complexity">
<span id="id1"></span><h1>Asymptotic Complexity<a class="headerlink" href="#asymptotic-complexity" title="Permalink to this heading"></a></h1>
<p>It is possible to calculate asymptotic complexity (Big O) from multiple runs of a benchmark. Run the
benchmark with different complexity N, then nanobench can calculate the best fitting curve.</p>
<p>The following example finds out the asymptotic complexity of <code class="docutils literal notranslate"><span class="pre">std::set</span></code>’s <code class="docutils literal notranslate"><span class="pre">find()</span></code>.</p>
<div class="literal-block-wrapper docutils container" id="id12">
<div class="code-block-caption"><span class="caption-text">tutorial_complexity_set.cpp</span><a class="headerlink" href="#id12" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;iostream&gt;</span><span class="cp"></span>
<span class="linenos"> 5</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;set&gt;</span><span class="cp"></span>
<span class="linenos"> 6</span>
<span class="linenos"> 7</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_complexity_set_find&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 8</span><span class="w">    </span><span class="c1">// Create a single benchmark instance that is used in multiple benchmark</span>
<span class="linenos"> 9</span><span class="w">    </span><span class="c1">// runs, with different settings for complexityN.</span>
<span class="linenos">10</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="w"> </span><span class="n">bench</span><span class="p">;</span><span class="w"></span>
<span class="linenos">11</span>
<span class="linenos">12</span><span class="w">    </span><span class="c1">// a RNG to generate input data</span>
<span class="linenos">13</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Rng</span><span class="w"> </span><span class="n">rng</span><span class="p">;</span><span class="w"></span>
<span class="linenos">14</span>
<span class="linenos">15</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">set</span><span class="o">&lt;</span><span class="kt">uint64_t</span><span class="o">&gt;</span><span class="w"> </span><span class="n">set</span><span class="p">;</span><span class="w"></span>
<span class="linenos">16</span>
<span class="linenos">17</span><span class="w">    </span><span class="c1">// Running the benchmark multiple times, with different number of elements</span>
<span class="linenos">18</span><span class="w">    </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="k">auto</span><span class="w"> </span><span class="n">setSize</span><span class="w"> </span><span class="o">:</span><span class="w"></span>
<span class="linenos">19</span><span class="w">         </span><span class="p">{</span><span class="mi">10U</span><span class="p">,</span><span class="w"> </span><span class="mi">20U</span><span class="p">,</span><span class="w"> </span><span class="mi">50U</span><span class="p">,</span><span class="w"> </span><span class="mi">100U</span><span class="p">,</span><span class="w"> </span><span class="mi">200U</span><span class="p">,</span><span class="w"> </span><span class="mi">500U</span><span class="p">,</span><span class="w"> </span><span class="mi">1000U</span><span class="p">,</span><span class="w"> </span><span class="mi">2000U</span><span class="p">,</span><span class="w"> </span><span class="mi">5000U</span><span class="p">,</span><span class="w"> </span><span class="mi">10000U</span><span class="p">})</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">20</span>
<span class="linenos">21</span><span class="w">        </span><span class="c1">// fill up the set with random data</span>
<span class="linenos">22</span><span class="w">        </span><span class="k">while</span><span class="w"> </span><span class="p">(</span><span class="n">set</span><span class="p">.</span><span class="n">size</span><span class="p">()</span><span class="w"> </span><span class="o">&lt;</span><span class="w"> </span><span class="n">setSize</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">23</span><span class="w">            </span><span class="n">set</span><span class="p">.</span><span class="n">insert</span><span class="p">(</span><span class="n">rng</span><span class="p">());</span><span class="w"></span>
<span class="linenos">24</span><span class="w">        </span><span class="p">}</span><span class="w"></span>
<span class="linenos">25</span>
<span class="linenos">26</span><span class="w">        </span><span class="c1">// Run the benchmark, provide setSize as the scaling variable.</span>
<span class="linenos">27</span><span class="w">        </span><span class="n">bench</span><span class="p">.</span><span class="n">complexityN</span><span class="p">(</span><span class="n">set</span><span class="p">.</span><span class="n">size</span><span class="p">()).</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;std::set find&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">28</span><span class="w">            </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">doNotOptimizeAway</span><span class="p">(</span><span class="n">set</span><span class="p">.</span><span class="n">find</span><span class="p">(</span><span class="n">rng</span><span class="p">()));</span><span class="w"></span>
<span class="linenos">29</span><span class="w">        </span><span class="p">});</span><span class="w"></span>
<span class="linenos">30</span><span class="w">    </span><span class="p">}</span><span class="w"></span>
<span class="linenos">31</span>
<span class="linenos">32</span><span class="w">    </span><span class="c1">// calculate BigO complexy best fit and print the results</span>
<span class="linenos">33</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">bench</span><span class="p">.</span><span class="n">complexityBigO</span><span class="p">()</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">endl</span><span class="p">;</span><span class="w"></span>
<span class="linenos">34</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>The loop runs the benchmark 10 times, with different set sizes from 10 to 10k.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Each of the 10 benchmark runs automatically scales the number of iterations so results are still
fast and accurate. In total the whole test takes about 90ms.</p>
</div>
<p>The <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench5BenchE" title="ankerl::nanobench::Bench"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">Bench</span></code></a> object holds the benchmark results of the 10 benchmark runs. Each benchmark is recorded with a
different setting for <a class="reference internal" href="reference.html#_CPPv4I0EN6ankerl9nanobench5Bench11complexityNER5Bench1T" title="ankerl::nanobench::Bench::complexityN"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">complexityN</span></code></a>.</p>
<p>After the benchmark prints the benchmark results, we calculate &amp; print the Big O of the most important complexity functions.
<code class="docutils literal notranslate"><span class="pre">std::cout</span> <span class="pre">&lt;&lt;</span> <span class="pre">bench.complexityBigO()</span> <span class="pre">&lt;&lt;</span> <span class="pre">std::endl;</span></code> prints e.g. this markdown table:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>|   coefficient |   err% | complexity
|--------------:|-------:|------------
|   6.66562e-09 |  29.1% | O(log n)
|   1.47588e-11 |  58.3% | O(n)
|   1.10742e-12 |  62.6% | O(n log n)
|   5.15683e-08 |  63.8% | O(1)
|   1.40387e-15 |  78.7% | O(n^2)
|   1.32792e-19 |  85.7% | O(n^3)
</pre></div>
</div>
<p>The table is sorted, best fitting complexity function first. So
<span class="math notranslate nohighlight">\(\mathcal{O}(\log{}n)\)</span> provides the best approximation for the complexity. Interestingly, in that case error compared to
<span class="math notranslate nohighlight">\(\mathcal{O}(n)\)</span> is not very large, which can be an indication that even though the red-black tree should theoretically have
logarithmic complexity, in practices that is not perfectly the case.</p>
</section>
<section id="rendering-mustache-like-templates">
<span id="templating"></span><h1>Rendering Mustache-like Templates<a class="headerlink" href="#rendering-mustache-like-templates" title="Permalink to this heading"></a></h1>
<p>Nanobench comes with a powerful <a class="reference external" href="https://mustache.github.io/">Mustache</a>-like template mechanism to process the benchmark
results into all kinds of formats. You can find a full description of all possible tags at <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench6renderEPKcRK5BenchRNSt7ostreamE" title="ankerl::nanobench::render"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ankerl::nanobench::render()</span></code></a>.</p>
<p>Several preconfigured format exist in the namespace <code class="docutils literal notranslate"><span class="pre">ankerl::nanobench::templates</span></code>. Rendering these templates can be done
with either <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench6renderEPKcRK5BenchRNSt7ostreamE" title="ankerl::nanobench::render"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ankerl::nanobench::render()</span></code></a>, or directly with <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench5Bench6renderEPKcRNSt7ostreamE" title="ankerl::nanobench::Bench::render"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ankerl::nanobench::Bench::render()</span></code></a>.</p>
<p>The following example shows how to use the <a class="reference internal" href="#csv-comma-separated-values">CSV - Comma-Separated Values</a> template, without writing the standard output.</p>
<div class="literal-block-wrapper docutils container" id="id13">
<div class="code-block-caption"><span class="caption-text">tutorial_render_simple.cpp</span><a class="headerlink" href="#id13" title="Permalink to this code"></a></div>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;atomic&gt;</span><span class="cp"></span>
<span class="linenos"> 5</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;iostream&gt;</span><span class="cp"></span>
<span class="linenos"> 6</span>
<span class="linenos"> 7</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_render_simple&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 8</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">atomic</span><span class="o">&lt;</span><span class="kt">int</span><span class="o">&gt;</span><span class="w"> </span><span class="n">x</span><span class="p">(</span><span class="mi">0</span><span class="p">);</span><span class="w"></span>
<span class="linenos"> 9</span>
<span class="linenos">10</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">()</span><span class="w"></span>
<span class="hll"><span class="linenos">11</span><span class="w">        </span><span class="p">.</span><span class="n">output</span><span class="p">(</span><span class="k">nullptr</span><span class="p">)</span><span class="w"></span>
</span><span class="linenos">12</span><span class="w">        </span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;std::vector&quot;</span><span class="p">,</span><span class="w"></span>
<span class="linenos">13</span><span class="w">             </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">14</span><span class="w">                 </span><span class="o">++</span><span class="n">x</span><span class="p">;</span><span class="w"></span>
<span class="linenos">15</span><span class="w">             </span><span class="p">})</span><span class="w"></span>
<span class="hll"><span class="linenos">16</span><span class="w">        </span><span class="p">.</span><span class="n">render</span><span class="p">(</span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">templates</span><span class="o">::</span><span class="n">csv</span><span class="p">(),</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="p">);</span><span class="w"></span>
</span><span class="linenos">17</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>In line 11 we call <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench5Bench6outputEPNSt7ostreamE" title="ankerl::nanobench::Bench::output"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">Bench::output()</span></code></a> with <code class="docutils literal notranslate"><span class="pre">nullptr</span></code>, thus disabling the standard output.</p>
<p>After the benchmark we directly call <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench5Bench6renderEPKcRNSt7ostreamE" title="ankerl::nanobench::Bench::render"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">Bench::render()</span></code></a> in line 16. Here we use the
CSV template, and write the rendered output to <code class="docutils literal notranslate"><span class="pre">std::cout</span></code>. When running, we get just the CSV output to the console which looks like this:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>&quot;title&quot;;&quot;name&quot;;&quot;unit&quot;;&quot;batch&quot;;&quot;elapsed&quot;;&quot;error %&quot;;&quot;instructions&quot;;&quot;branches&quot;;&quot;branch misses&quot;;&quot;total&quot;
&quot;benchmark&quot;;&quot;std::vector&quot;;&quot;op&quot;;1;6.51982200647249e-09;8.26465858909014e-05;23.0034662045061;5;0.00116867939228672;0.000171959
</pre></div>
</div>
<p>Nanobench comes with a few preconfigured templates, residing in the namespace <code class="docutils literal notranslate"><span class="pre">ankerl::nanobench::templates</span></code>. To demonstrate what these templates can do,
here is a simple example that benchmarks two random generators <code class="docutils literal notranslate"><span class="pre">std::mt19937_64</span></code> and <code class="docutils literal notranslate"><span class="pre">std::knuth_b</span></code> and prints both the template and the rendered
output:</p>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;fstream&gt;</span><span class="cp"></span>
<span class="linenos"> 5</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;random&gt;</span><span class="cp"></span>
<span class="linenos"> 6</span>
<span class="linenos"> 7</span><span class="k">namespace</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 8</span>
<span class="linenos"> 9</span><span class="kt">void</span><span class="w"> </span><span class="nf">gen</span><span class="p">(</span><span class="n">std</span><span class="o">::</span><span class="n">string</span><span class="w"> </span><span class="k">const</span><span class="o">&amp;</span><span class="w"> </span><span class="n">typeName</span><span class="p">,</span><span class="w"> </span><span class="kt">char</span><span class="w"> </span><span class="k">const</span><span class="o">*</span><span class="w"> </span><span class="n">mustacheTemplate</span><span class="p">,</span><span class="w"></span>
<span class="linenos">10</span><span class="w">         </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="w"> </span><span class="k">const</span><span class="o">&amp;</span><span class="w"> </span><span class="n">bench</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">11</span>
<span class="linenos">12</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">ofstream</span><span class="w"> </span><span class="n">templateOut</span><span class="p">(</span><span class="s">&quot;mustache.template.&quot;</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">typeName</span><span class="p">);</span><span class="w"></span>
<span class="linenos">13</span><span class="w">    </span><span class="n">templateOut</span><span class="w"> </span><span class="o">&lt;&lt;</span><span class="w"> </span><span class="n">mustacheTemplate</span><span class="p">;</span><span class="w"></span>
<span class="linenos">14</span>
<span class="linenos">15</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">ofstream</span><span class="w"> </span><span class="n">renderOut</span><span class="p">(</span><span class="s">&quot;mustache.render.&quot;</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">typeName</span><span class="p">);</span><span class="w"></span>
<span class="linenos">16</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">render</span><span class="p">(</span><span class="n">mustacheTemplate</span><span class="p">,</span><span class="w"> </span><span class="n">bench</span><span class="p">,</span><span class="w"> </span><span class="n">renderOut</span><span class="p">);</span><span class="w"></span>
<span class="linenos">17</span><span class="p">}</span><span class="w"></span>
<span class="linenos">18</span>
<span class="linenos">19</span><span class="p">}</span><span class="w"> </span><span class="c1">// namespace</span>
<span class="linenos">20</span>
<span class="linenos">21</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_mustache&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">22</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="w"> </span><span class="n">bench</span><span class="p">;</span><span class="w"></span>
<span class="linenos">23</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">title</span><span class="p">(</span><span class="s">&quot;Benchmarking std::mt19937_64 and std::knuth_b&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">24</span>
<span class="linenos">25</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">mt19937_64</span><span class="w"> </span><span class="n">rng1</span><span class="p">;</span><span class="w"></span>
<span class="linenos">26</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;std::mt19937_64&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">27</span><span class="w">        </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">doNotOptimizeAway</span><span class="p">(</span><span class="n">rng1</span><span class="p">());</span><span class="w"></span>
<span class="linenos">28</span><span class="w">    </span><span class="p">});</span><span class="w"></span>
<span class="linenos">29</span>
<span class="linenos">30</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">knuth_b</span><span class="w"> </span><span class="n">rng2</span><span class="p">;</span><span class="w"></span>
<span class="linenos">31</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;std::knuth_b&quot;</span><span class="p">,</span><span class="w"> </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">32</span><span class="w">        </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">doNotOptimizeAway</span><span class="p">(</span><span class="n">rng2</span><span class="p">());</span><span class="w"></span>
<span class="linenos">33</span><span class="w">    </span><span class="p">});</span><span class="w"></span>
<span class="linenos">34</span>
<span class="linenos">35</span><span class="w">    </span><span class="n">gen</span><span class="p">(</span><span class="s">&quot;json&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">templates</span><span class="o">::</span><span class="n">json</span><span class="p">(),</span><span class="w"> </span><span class="n">bench</span><span class="p">);</span><span class="w"></span>
<span class="linenos">36</span><span class="w">    </span><span class="n">gen</span><span class="p">(</span><span class="s">&quot;html&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">templates</span><span class="o">::</span><span class="n">htmlBoxplot</span><span class="p">(),</span><span class="w"> </span><span class="n">bench</span><span class="p">);</span><span class="w"></span>
<span class="linenos">37</span><span class="w">    </span><span class="n">gen</span><span class="p">(</span><span class="s">&quot;csv&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">templates</span><span class="o">::</span><span class="n">csv</span><span class="p">(),</span><span class="w"> </span><span class="n">bench</span><span class="p">);</span><span class="w"></span>
<span class="linenos">38</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
<p>Nanobench allows to specify further context information, which may be accessed using <code class="docutils literal notranslate"><span class="pre">{{context(name)}}</span></code> where <code class="docutils literal notranslate"><span class="pre">name</span></code> names a variable defined via <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench5Bench7contextEPKcPKc" title="ankerl::nanobench::Bench::context"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">Bench::context()</span></code></a>.</p>
<div class="highlight-c++ notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;iostream&gt;</span><span class="cp"></span>
<span class="linenos"> 5</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;cmath&gt;</span><span class="cp"></span>
<span class="linenos"> 6</span>
<span class="linenos"> 7</span><span class="k">namespace</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 8</span>
<span class="linenos"> 9</span><span class="k">template</span><span class="o">&lt;</span><span class="k">typename</span><span class="w"> </span><span class="nc">T</span><span class="o">&gt;</span><span class="w"></span>
<span class="linenos">10</span><span class="kt">void</span><span class="w"> </span><span class="n">fma</span><span class="p">()</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">11</span><span class="w">    </span><span class="n">T</span><span class="w"> </span><span class="nf">x</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span><span class="w"> </span><span class="n">y</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span><span class="w"> </span><span class="n">z</span><span class="p">(</span><span class="mi">3</span><span class="p">);</span><span class="w"></span>
<span class="linenos">12</span><span class="w">    </span><span class="n">z</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">fma</span><span class="p">(</span><span class="n">x</span><span class="p">,</span><span class="w"> </span><span class="n">y</span><span class="p">,</span><span class="w"> </span><span class="n">z</span><span class="p">);</span><span class="w"></span>
<span class="linenos">13</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">doNotOptimizeAway</span><span class="p">(</span><span class="n">z</span><span class="p">);</span><span class="w"></span>
<span class="linenos">14</span><span class="p">}</span><span class="w"></span>
<span class="linenos">15</span>
<span class="linenos">16</span><span class="k">template</span><span class="o">&lt;</span><span class="k">typename</span><span class="w"> </span><span class="nc">T</span><span class="o">&gt;</span><span class="w"></span>
<span class="linenos">17</span><span class="kt">void</span><span class="w"> </span><span class="n">plus_eq</span><span class="p">()</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">18</span><span class="w">    </span><span class="n">T</span><span class="w"> </span><span class="nf">x</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span><span class="w"> </span><span class="n">y</span><span class="p">(</span><span class="mi">2</span><span class="p">),</span><span class="w"> </span><span class="n">z</span><span class="p">(</span><span class="mi">3</span><span class="p">);</span><span class="w"></span>
<span class="linenos">19</span><span class="w">    </span><span class="n">z</span><span class="w"> </span><span class="o">+=</span><span class="w"> </span><span class="n">x</span><span class="o">*</span><span class="n">y</span><span class="p">;</span><span class="w"></span>
<span class="linenos">20</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">doNotOptimizeAway</span><span class="p">(</span><span class="n">z</span><span class="p">);</span><span class="w"></span>
<span class="linenos">21</span><span class="p">}</span><span class="w"></span>
<span class="linenos">22</span>
<span class="linenos">23</span><span class="kt">char</span><span class="w"> </span><span class="k">const</span><span class="o">*</span><span class="w"> </span><span class="n">csv</span><span class="p">()</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">24</span><span class="w">    </span><span class="k">return</span><span class="w"> </span><span class="sa">R</span><span class="s">&quot;</span><span class="dl">DELIM(</span><span class="s">&quot;title&quot;;&quot;name&quot;;&quot;scalar&quot;;&quot;foo&quot;;&quot;elapsed&quot;;&quot;total&quot;</span>
<span class="linenos">25</span><span class="s">{{#result}}&quot;{{title}}&quot;;&quot;{{name}}&quot;;&quot;{{context(scalar)}}&quot;;&quot;{{context(foo)}}&quot;;{{median(elapsed)}};{{sumProduct(iterations, elapsed)}}</span>
<span class="linenos">26</span><span class="s">{{/result}}</span><span class="dl">)DELIM</span><span class="s">&quot;</span><span class="p">;</span><span class="w"></span>
<span class="linenos">27</span><span class="p">}</span><span class="w"></span>
<span class="linenos">28</span>
<span class="linenos">29</span><span class="p">}</span><span class="w"> </span><span class="c1">// namespace</span>
<span class="linenos">30</span>
<span class="linenos">31</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;tutorial_context&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">32</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="w"> </span><span class="n">bench</span><span class="p">;</span><span class="w"></span>
<span class="linenos">33</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">title</span><span class="p">(</span><span class="s">&quot;Addition&quot;</span><span class="p">).</span><span class="n">output</span><span class="p">(</span><span class="k">nullptr</span><span class="p">);</span><span class="w"></span>
<span class="linenos">34</span><span class="w">    </span><span class="n">bench</span><span class="w"></span>
<span class="linenos">35</span><span class="w">        </span><span class="p">.</span><span class="n">context</span><span class="p">(</span><span class="s">&quot;scalar&quot;</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;f32&quot;</span><span class="p">)</span><span class="w"></span>
<span class="linenos">36</span><span class="w">        </span><span class="p">.</span><span class="n">context</span><span class="p">(</span><span class="s">&quot;foo&quot;</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;bar&quot;</span><span class="p">)</span><span class="w"></span>
<span class="linenos">37</span><span class="w">        </span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;+=&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">plus_eq</span><span class="o">&lt;</span><span class="kt">float</span><span class="o">&gt;</span><span class="p">)</span><span class="w"></span>
<span class="linenos">38</span><span class="w">        </span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;fma&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">fma</span><span class="o">&lt;</span><span class="kt">float</span><span class="o">&gt;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">39</span><span class="w">    </span><span class="n">bench</span><span class="w"></span>
<span class="linenos">40</span><span class="w">        </span><span class="p">.</span><span class="n">context</span><span class="p">(</span><span class="s">&quot;scalar&quot;</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;f64&quot;</span><span class="p">)</span><span class="w"></span>
<span class="linenos">41</span><span class="w">        </span><span class="p">.</span><span class="n">context</span><span class="p">(</span><span class="s">&quot;foo&quot;</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;baz&quot;</span><span class="p">)</span><span class="w"></span>
<span class="linenos">42</span><span class="w">        </span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;+=&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">plus_eq</span><span class="o">&lt;</span><span class="kt">double</span><span class="o">&gt;</span><span class="p">)</span><span class="w"></span>
<span class="linenos">43</span><span class="w">        </span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;fma&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">fma</span><span class="o">&lt;</span><span class="kt">double</span><span class="o">&gt;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">44</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">render</span><span class="p">(</span><span class="n">csv</span><span class="p">(),</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="p">);</span><span class="w"></span>
<span class="linenos">45</span><span class="w">    </span><span class="c1">// Changing the title resets the results, but not the context:</span>
<span class="linenos">46</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">title</span><span class="p">(</span><span class="s">&quot;New Title&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">47</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;+=&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">plus_eq</span><span class="o">&lt;</span><span class="kt">float</span><span class="o">&gt;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">48</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">render</span><span class="p">(</span><span class="n">csv</span><span class="p">(),</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="p">);</span><span class="w"></span>
<span class="linenos">49</span><span class="w">    </span><span class="n">CHECK_EQ</span><span class="p">(</span><span class="n">bench</span><span class="p">.</span><span class="n">results</span><span class="p">().</span><span class="n">front</span><span class="p">().</span><span class="n">context</span><span class="p">(</span><span class="s">&quot;foo&quot;</span><span class="p">),</span><span class="w"> </span><span class="s">&quot;baz&quot;</span><span class="p">);</span><span class="w"> </span><span class="c1">// != bar</span>
<span class="linenos">50</span><span class="w">    </span><span class="c1">// The context has to be reset manually, which causes render to fail:</span>
<span class="linenos">51</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">title</span><span class="p">(</span><span class="s">&quot;Yet Another Title&quot;</span><span class="p">).</span><span class="n">clearContext</span><span class="p">();</span><span class="w"></span>
<span class="linenos">52</span><span class="w">    </span><span class="n">bench</span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;+=&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">plus_eq</span><span class="o">&lt;</span><span class="kt">float</span><span class="o">&gt;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">53</span><span class="w">    </span><span class="n">CHECK_THROWS</span><span class="p">(</span><span class="n">bench</span><span class="p">.</span><span class="n">render</span><span class="p">(</span><span class="n">csv</span><span class="p">(),</span><span class="w"> </span><span class="n">std</span><span class="o">::</span><span class="n">cout</span><span class="p">));</span><span class="w"></span>
<span class="linenos">54</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
<section id="csv-comma-separated-values">
<span id="tutorial-template-csv"></span><h2>CSV - Comma-Separated Values<a class="headerlink" href="#csv-comma-separated-values" title="Permalink to this heading"></a></h2>
<p>The function <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench9templates3csvEv" title="ankerl::nanobench::templates::csv"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ankerl::nanobench::templates::csv()</span></code></a> provides this template:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><span class="linenos">1</span>&quot;title&quot;;&quot;name&quot;;&quot;unit&quot;;&quot;batch&quot;;&quot;elapsed&quot;;&quot;error %&quot;;&quot;instructions&quot;;&quot;branches&quot;;&quot;branch misses&quot;;&quot;total&quot;
<span class="linenos">2</span>{{#result}}&quot;{{title}}&quot;;&quot;{{name}}&quot;;&quot;{{unit}}&quot;;{{batch}};{{median(elapsed)}};{{medianAbsolutePercentError(elapsed)}};{{median(instructions)}};{{median(branchinstructions)}};{{median(branchmisses)}};{{sumProduct(iterations, elapsed)}}
<span class="linenos">3</span>{{/result}}
</pre></div>
</div>
<p>This generates a compact CSV file, where entries are separated by a semicolon <cite>;</cite>. Run with the example, I get this output:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><span class="linenos">1</span>&quot;title&quot;;&quot;name&quot;;&quot;unit&quot;;&quot;batch&quot;;&quot;elapsed&quot;;&quot;error %&quot;;&quot;instructions&quot;;&quot;branches&quot;;&quot;branch misses&quot;;&quot;total&quot;
<span class="linenos">2</span>&quot;Benchmarking std::mt19937_64 and std::knuth_b&quot;;&quot;std::mt19937_64&quot;;&quot;op&quot;;1;2.54441805225653e-08;0.0236579384033733;125.989678899083;16.7645714285714;0.564133016627078;0.000218811
<span class="linenos">3</span>&quot;Benchmarking std::mt19937_64 and std::knuth_b&quot;;&quot;std::knuth_b&quot;;&quot;op&quot;;1;3.19013867488444e-08;0.00091350764819687;170.013008130081;28;0.0031104199066874;0.000217248
</pre></div>
</div>
<p>Rendered as CSV table:</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 10%" />
<col style="width: 10%" />
<col style="width: 10%" />
<col style="width: 10%" />
<col style="width: 10%" />
<col style="width: 10%" />
<col style="width: 10%" />
<col style="width: 10%" />
<col style="width: 10%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>title</p></th>
<th class="head"><p>name</p></th>
<th class="head"><p>unit</p></th>
<th class="head"><p>batch</p></th>
<th class="head"><p>elapsed</p></th>
<th class="head"><p>error %</p></th>
<th class="head"><p>instructions</p></th>
<th class="head"><p>branches</p></th>
<th class="head"><p>branch misses</p></th>
<th class="head"><p>total</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>Benchmarking std::mt19937_64 and std::knuth_b</p></td>
<td><p>std::mt19937_64</p></td>
<td><p>op</p></td>
<td><p>1</p></td>
<td><p>2.54441805225653e-08</p></td>
<td><p>0.0236579384033733</p></td>
<td><p>125.989678899083</p></td>
<td><p>16.7645714285714</p></td>
<td><p>0.564133016627078</p></td>
<td><p>0.000218811</p></td>
</tr>
<tr class="row-odd"><td><p>Benchmarking std::mt19937_64 and std::knuth_b</p></td>
<td><p>std::knuth_b</p></td>
<td><p>op</p></td>
<td><p>1</p></td>
<td><p>3.19013867488444e-08</p></td>
<td><p>0.00091350764819687</p></td>
<td><p>170.013008130081</p></td>
<td><p>28</p></td>
<td><p>0.0031104199066874</p></td>
<td><p>0.000217248</p></td>
</tr>
</tbody>
</table>
<p>Note that the CSV template doesn’t provide all the data that is available.</p>
</section>
<section id="html-box-plots">
<span id="tutorial-template-html"></span><h2>HTML Box Plots<a class="headerlink" href="#html-box-plots" title="Permalink to this heading"></a></h2>
<p>With the template <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench9templates11htmlBoxplotEv" title="ankerl::nanobench::templates::htmlBoxplot"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ankerl::nanobench::templates::htmlBoxplot()</span></code></a> you get a <a class="reference external" href="https://plotly.com/javascript/">plotly</a> based HTML output which generates
a boxplot of the runtime. The template is rather simple.</p>
<div class="highlight-xml notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="nt">&lt;html&gt;</span>
<span class="linenos"> 2</span>
<span class="linenos"> 3</span><span class="nt">&lt;head&gt;</span>
<span class="linenos"> 4</span>    <span class="nt">&lt;script</span> <span class="na">src=</span><span class="s">&quot;https://cdn.plot.ly/plotly-latest.min.js&quot;</span><span class="nt">&gt;&lt;/script&gt;</span>
<span class="linenos"> 5</span><span class="nt">&lt;/head&gt;</span>
<span class="linenos"> 6</span>
<span class="linenos"> 7</span><span class="nt">&lt;body&gt;</span>
<span class="linenos"> 8</span>    <span class="nt">&lt;div</span> <span class="na">id=</span><span class="s">&quot;myDiv&quot;</span><span class="nt">&gt;&lt;/div&gt;</span>
<span class="linenos"> 9</span>    <span class="nt">&lt;script&gt;</span>
<span class="linenos">10</span>        var data = [
<span class="linenos">11</span>            {{#result}}{
<span class="linenos">12</span>                name: &#39;{{name}}&#39;,
<span class="linenos">13</span>                y: [{{#measurement}}{{elapsed}}{{^-last}}, {{/last}}{{/measurement}}],
<span class="linenos">14</span>            },
<span class="linenos">15</span>            {{/result}}
<span class="linenos">16</span>        ];
<span class="linenos">17</span>        var title = &#39;{{title}}&#39;;
<span class="linenos">18</span>
<span class="linenos">19</span>        data = data.map(a =&gt; Object.assign(a, { boxpoints: &#39;all&#39;, pointpos: 0, type: &#39;box&#39; }));
<span class="linenos">20</span>        var layout = { title: { text: title }, showlegend: false, yaxis: { title: &#39;time per unit&#39;, rangemode: &#39;tozero&#39;, autorange: true } }; Plotly.newPlot(&#39;myDiv&#39;, data, layout, {responsive: true});
<span class="linenos">21</span>    <span class="nt">&lt;/script&gt;</span>
<span class="linenos">22</span><span class="nt">&lt;/body&gt;</span>
<span class="linenos">23</span>
<span class="linenos">24</span><span class="nt">&lt;/html&gt;</span>
</pre></div>
</div>
<p>This generates a nice interactive boxplot, which gives a nice visual showcase of the runtime performance of the evaluated benchmarks. Each epoch is visualized as a dot,
and the boxplot itself shows median, percentiles, and outliers. You’ll might want to increase the default number of epochs for an even better visualization result.</p>
<html>

<head>
    <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
</head>

<body>
    <div id="myDiv"></div>
    <script>
        var data = [
            {
                name: 'std::mt19937_64',
                y: [2.54708571428571e-08, 2.58467243510507e-08, 2.24097693351425e-08, 2.53405963302752e-08, 2.59256594724221e-08, 2.25310880829016e-08, 2.54441805225653e-08, 2.20126262626263e-08, 2.63870541611625e-08, 2.23305882352941e-08, 2.60607235142119e-08],
            },
            {
                name: 'std::knuth_b',
                y: [3.2137323943662e-08, 3.19305555555556e-08, 3.18973561430793e-08, 3.1912013536379e-08, 3.19049034175334e-08, 3.19013867488444e-08, 3.18547854785479e-08, 3.18769230769231e-08, 3.18520325203252e-08, 3.18618307426598e-08, 3.19558599695586e-08],
            },
            
        ];
        var title = 'Benchmarking std::mt19937_64 and std::knuth_b';

        data = data.map(a => Object.assign(a, { boxpoints: 'all', pointpos: 0, type: 'box' }));
        var layout = { title: { text: title }, showlegend: false, yaxis: { title: 'time per unit', rangemode: 'tozero', autorange: true } }; Plotly.newPlot('myDiv', data, layout, {responsive: true});
    </script>
</body>

</html></section>
<section id="json-javascript-object-notation">
<span id="tutorial-template-json"></span><h2>JSON - JavaScript Object Notation<a class="headerlink" href="#json-javascript-object-notation" title="Permalink to this heading"></a></h2>
<p>The <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench9templates4jsonEv" title="ankerl::nanobench::templates::json"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ankerl::nanobench::templates::json()</span></code></a> template gives everything, all data that is available, from all runs. The template is therefore quite complex:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span>{
<span class="linenos"> 2</span>    &quot;results&quot;: [
<span class="linenos"> 3</span>{{#result}}        {
<span class="linenos"> 4</span>            &quot;title&quot;: &quot;{{title}}&quot;,
<span class="linenos"> 5</span>            &quot;name&quot;: &quot;{{name}}&quot;,
<span class="linenos"> 6</span>            &quot;unit&quot;: &quot;{{unit}}&quot;,
<span class="linenos"> 7</span>            &quot;batch&quot;: {{batch}},
<span class="linenos"> 8</span>            &quot;complexityN&quot;: {{complexityN}},
<span class="linenos"> 9</span>            &quot;epochs&quot;: {{epochs}},
<span class="linenos">10</span>            &quot;clockResolution&quot;: {{clockResolution}},
<span class="linenos">11</span>            &quot;clockResolutionMultiple&quot;: {{clockResolutionMultiple}},
<span class="linenos">12</span>            &quot;maxEpochTime&quot;: {{maxEpochTime}},
<span class="linenos">13</span>            &quot;minEpochTime&quot;: {{minEpochTime}},
<span class="linenos">14</span>            &quot;minEpochIterations&quot;: {{minEpochIterations}},
<span class="linenos">15</span>            &quot;epochIterations&quot;: {{epochIterations}},
<span class="linenos">16</span>            &quot;warmup&quot;: {{warmup}},
<span class="linenos">17</span>            &quot;relative&quot;: {{relative}},
<span class="linenos">18</span>            &quot;median(elapsed)&quot;: {{median(elapsed)}},
<span class="linenos">19</span>            &quot;medianAbsolutePercentError(elapsed)&quot;: {{medianAbsolutePercentError(elapsed)}},
<span class="linenos">20</span>            &quot;median(instructions)&quot;: {{median(instructions)}},
<span class="linenos">21</span>            &quot;medianAbsolutePercentError(instructions)&quot;: {{medianAbsolutePercentError(instructions)}},
<span class="linenos">22</span>            &quot;median(cpucycles)&quot;: {{median(cpucycles)}},
<span class="linenos">23</span>            &quot;median(contextswitches)&quot;: {{median(contextswitches)}},
<span class="linenos">24</span>            &quot;median(pagefaults)&quot;: {{median(pagefaults)}},
<span class="linenos">25</span>            &quot;median(branchinstructions)&quot;: {{median(branchinstructions)}},
<span class="linenos">26</span>            &quot;median(branchmisses)&quot;: {{median(branchmisses)}},
<span class="linenos">27</span>            &quot;totalTime&quot;: {{sumProduct(iterations, elapsed)}},
<span class="linenos">28</span>            &quot;measurements&quot;: [
<span class="linenos">29</span>{{#measurement}}                {
<span class="linenos">30</span>                    &quot;iterations&quot;: {{iterations}},
<span class="linenos">31</span>                    &quot;elapsed&quot;: {{elapsed}},
<span class="linenos">32</span>                    &quot;pagefaults&quot;: {{pagefaults}},
<span class="linenos">33</span>                    &quot;cpucycles&quot;: {{cpucycles}},
<span class="linenos">34</span>                    &quot;contextswitches&quot;: {{contextswitches}},
<span class="linenos">35</span>                    &quot;instructions&quot;: {{instructions}},
<span class="linenos">36</span>                    &quot;branchinstructions&quot;: {{branchinstructions}},
<span class="linenos">37</span>                    &quot;branchmisses&quot;: {{branchmisses}}
<span class="linenos">38</span>                }{{^-last}},{{/-last}}
<span class="linenos">39</span>{{/measurement}}            ]
<span class="linenos">40</span>        }{{^-last}},{{/-last}}
<span class="linenos">41</span>{{/result}}    ]
<span class="linenos">42</span>}
</pre></div>
</div>
<p>This also gives the data from each separate <a class="reference internal" href="reference.html#_CPPv4N6ankerl9nanobench5Bench6epochsE6size_t" title="ankerl::nanobench::Bench::epochs"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">ankerl::nanobench::Bench::epochs()</span></code></a>, not just the accumulated data as in the CSV template.</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="linenos">  1</span><span class="p">{</span><span class="w"></span>
<span class="linenos">  2</span><span class="w">    </span><span class="nt">&quot;results&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w"></span>
<span class="linenos">  3</span><span class="w">        </span><span class="p">{</span><span class="w"></span>
<span class="linenos">  4</span><span class="w">            </span><span class="nt">&quot;title&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;Benchmarking std::mt19937_64 and std::knuth_b&quot;</span><span class="p">,</span><span class="w"></span>
<span class="linenos">  5</span><span class="w">            </span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;std::mt19937_64&quot;</span><span class="p">,</span><span class="w"></span>
<span class="linenos">  6</span><span class="w">            </span><span class="nt">&quot;unit&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;op&quot;</span><span class="p">,</span><span class="w"></span>
<span class="linenos">  7</span><span class="w">            </span><span class="nt">&quot;batch&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w"></span>
<span class="linenos">  8</span><span class="w">            </span><span class="nt">&quot;complexityN&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">-1</span><span class="p">,</span><span class="w"></span>
<span class="linenos">  9</span><span class="w">            </span><span class="nt">&quot;epochs&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">11</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 10</span><span class="w">            </span><span class="nt">&quot;clockResolution&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">1.8e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 11</span><span class="w">            </span><span class="nt">&quot;clockResolutionMultiple&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1000</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 12</span><span class="w">            </span><span class="nt">&quot;maxEpochTime&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.1</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 13</span><span class="w">            </span><span class="nt">&quot;minEpochTime&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 14</span><span class="w">            </span><span class="nt">&quot;minEpochIterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 15</span><span class="w">            </span><span class="nt">&quot;warmup&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 16</span><span class="w">            </span><span class="nt">&quot;relative&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 17</span><span class="w">            </span><span class="nt">&quot;median(elapsed)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.54441805225653e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 18</span><span class="w">            </span><span class="nt">&quot;medianAbsolutePercentError(elapsed)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.0236579384033733</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 19</span><span class="w">            </span><span class="nt">&quot;median(instructions)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">125.989678899083</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 20</span><span class="w">            </span><span class="nt">&quot;medianAbsolutePercentError(instructions)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.035125448044942</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 21</span><span class="w">            </span><span class="nt">&quot;median(cpucycles)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">81.3479809976247</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 22</span><span class="w">            </span><span class="nt">&quot;median(contextswitches)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 23</span><span class="w">            </span><span class="nt">&quot;median(pagefaults)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 24</span><span class="w">            </span><span class="nt">&quot;median(branchinstructions)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">16.7645714285714</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 25</span><span class="w">            </span><span class="nt">&quot;median(branchmisses)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.564133016627078</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 26</span><span class="w">            </span><span class="nt">&quot;totalTime&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.000218811</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 27</span><span class="w">            </span><span class="nt">&quot;measurements&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w"></span>
<span class="linenos"> 28</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 29</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">875</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 30</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.54708571428571e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 31</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 32</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">81.472</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 33</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 34</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">125.885714285714</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 35</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">16.7645714285714</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 36</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.574857142857143</span><span class="w"></span>
<span class="linenos"> 37</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos"> 38</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 39</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">809</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 40</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.58467243510507e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 41</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 42</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">82.5290482076638</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 43</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 44</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">128.771322620519</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 45</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">17.0296662546354</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 46</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.582200247218789</span><span class="w"></span>
<span class="linenos"> 47</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos"> 48</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 49</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">737</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 50</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.24097693351425e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 51</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 52</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">71.6431478968792</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 53</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 54</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">118.374491180461</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 55</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">15.9470827679783</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 56</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.417910447761194</span><span class="w"></span>
<span class="linenos"> 57</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos"> 58</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 59</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">872</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 60</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.53405963302752e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 61</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 62</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">80.9896788990826</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 63</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 64</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">125.989678899083</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 65</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">16.7580275229358</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 66</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.563073394495413</span><span class="w"></span>
<span class="linenos"> 67</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos"> 68</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 69</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">834</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 70</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.59256594724221e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 71</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 72</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">82.7661870503597</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 73</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 74</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">127.635491606715</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 75</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">16.9352517985612</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 76</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.575539568345324</span><span class="w"></span>
<span class="linenos"> 77</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos"> 78</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 79</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">772</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 80</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.25310880829016e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 81</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 82</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">72.0129533678757</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 83</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 84</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">117.108808290155</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 85</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">15.8341968911917</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 86</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.405440414507772</span><span class="w"></span>
<span class="linenos"> 87</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos"> 88</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 89</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">842</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 90</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.54441805225653e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 91</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 92</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">81.3479809976247</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 93</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 94</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">127.266033254157</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 95</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">16.8859857482185</span><span class="p">,</span><span class="w"></span>
<span class="linenos"> 96</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.564133016627078</span><span class="w"></span>
<span class="linenos"> 97</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos"> 98</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 99</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">792</span><span class="p">,</span><span class="w"></span>
<span class="linenos">100</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.20126262626263e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">101</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">102</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">70.3623737373737</span><span class="p">,</span><span class="w"></span>
<span class="linenos">103</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">104</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">116.420454545455</span><span class="p">,</span><span class="w"></span>
<span class="linenos">105</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">15.7588383838384</span><span class="p">,</span><span class="w"></span>
<span class="linenos">106</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.396464646464646</span><span class="w"></span>
<span class="linenos">107</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">108</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">109</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">757</span><span class="p">,</span><span class="w"></span>
<span class="linenos">110</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.63870541611625e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">111</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">112</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">84.332892998679</span><span class="p">,</span><span class="w"></span>
<span class="linenos">113</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">114</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">131.462351387054</span><span class="p">,</span><span class="w"></span>
<span class="linenos">115</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">17.334214002642</span><span class="p">,</span><span class="w"></span>
<span class="linenos">116</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.618229854689564</span><span class="w"></span>
<span class="linenos">117</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">118</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">119</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">850</span><span class="p">,</span><span class="w"></span>
<span class="linenos">120</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.23305882352941e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">121</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">122</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">71.3505882352941</span><span class="p">,</span><span class="w"></span>
<span class="linenos">123</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">124</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">114.629411764706</span><span class="p">,</span><span class="w"></span>
<span class="linenos">125</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">15.5823529411765</span><span class="p">,</span><span class="w"></span>
<span class="linenos">126</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.392941176470588</span><span class="w"></span>
<span class="linenos">127</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">128</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">129</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">774</span><span class="p">,</span><span class="w"></span>
<span class="linenos">130</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">2.60607235142119e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">131</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">132</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">83.1679586563308</span><span class="p">,</span><span class="w"></span>
<span class="linenos">133</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">134</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">130.576227390181</span><span class="p">,</span><span class="w"></span>
<span class="linenos">135</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">17.2635658914729</span><span class="p">,</span><span class="w"></span>
<span class="linenos">136</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.590439276485788</span><span class="w"></span>
<span class="linenos">137</span><span class="w">                </span><span class="p">}</span><span class="w"></span>
<span class="linenos">138</span><span class="w">            </span><span class="p">]</span><span class="w"></span>
<span class="linenos">139</span><span class="w">        </span><span class="p">},</span><span class="w"></span>
<span class="linenos">140</span><span class="w">        </span><span class="p">{</span><span class="w"></span>
<span class="linenos">141</span><span class="w">            </span><span class="nt">&quot;title&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;Benchmarking std::mt19937_64 and std::knuth_b&quot;</span><span class="p">,</span><span class="w"></span>
<span class="linenos">142</span><span class="w">            </span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;std::knuth_b&quot;</span><span class="p">,</span><span class="w"></span>
<span class="linenos">143</span><span class="w">            </span><span class="nt">&quot;unit&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;op&quot;</span><span class="p">,</span><span class="w"></span>
<span class="linenos">144</span><span class="w">            </span><span class="nt">&quot;batch&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w"></span>
<span class="linenos">145</span><span class="w">            </span><span class="nt">&quot;complexityN&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">-1</span><span class="p">,</span><span class="w"></span>
<span class="linenos">146</span><span class="w">            </span><span class="nt">&quot;epochs&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">11</span><span class="p">,</span><span class="w"></span>
<span class="linenos">147</span><span class="w">            </span><span class="nt">&quot;clockResolution&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">1.8e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">148</span><span class="w">            </span><span class="nt">&quot;clockResolutionMultiple&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1000</span><span class="p">,</span><span class="w"></span>
<span class="linenos">149</span><span class="w">            </span><span class="nt">&quot;maxEpochTime&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.1</span><span class="p">,</span><span class="w"></span>
<span class="linenos">150</span><span class="w">            </span><span class="nt">&quot;minEpochTime&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">151</span><span class="w">            </span><span class="nt">&quot;minEpochIterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w"></span>
<span class="linenos">152</span><span class="w">            </span><span class="nt">&quot;warmup&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">153</span><span class="w">            </span><span class="nt">&quot;relative&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">154</span><span class="w">            </span><span class="nt">&quot;median(elapsed)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.19013867488444e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">155</span><span class="w">            </span><span class="nt">&quot;medianAbsolutePercentError(elapsed)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.00091350764819687</span><span class="p">,</span><span class="w"></span>
<span class="linenos">156</span><span class="w">            </span><span class="nt">&quot;median(instructions)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.013008130081</span><span class="p">,</span><span class="w"></span>
<span class="linenos">157</span><span class="w">            </span><span class="nt">&quot;medianAbsolutePercentError(instructions)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">4.11992392254248e-06</span><span class="p">,</span><span class="w"></span>
<span class="linenos">158</span><span class="w">            </span><span class="nt">&quot;median(cpucycles)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">101.973254086181</span><span class="p">,</span><span class="w"></span>
<span class="linenos">159</span><span class="w">            </span><span class="nt">&quot;median(contextswitches)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">160</span><span class="w">            </span><span class="nt">&quot;median(pagefaults)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">161</span><span class="w">            </span><span class="nt">&quot;median(branchinstructions)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">162</span><span class="w">            </span><span class="nt">&quot;median(branchmisses)&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.0031104199066874</span><span class="p">,</span><span class="w"></span>
<span class="linenos">163</span><span class="w">            </span><span class="nt">&quot;totalTime&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.000217248</span><span class="p">,</span><span class="w"></span>
<span class="linenos">164</span><span class="w">            </span><span class="nt">&quot;measurements&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w"></span>
<span class="linenos">165</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">166</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">568</span><span class="p">,</span><span class="w"></span>
<span class="linenos">167</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.2137323943662e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">168</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">169</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">102.55985915493</span><span class="p">,</span><span class="w"></span>
<span class="linenos">170</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">171</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.014084507042</span><span class="p">,</span><span class="w"></span>
<span class="linenos">172</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">173</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.00528169014084507</span><span class="w"></span>
<span class="linenos">174</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">175</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">176</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">576</span><span class="p">,</span><span class="w"></span>
<span class="linenos">177</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.19305555555556e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">178</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">179</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">102.059027777778</span><span class="p">,</span><span class="w"></span>
<span class="linenos">180</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">181</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.013888888889</span><span class="p">,</span><span class="w"></span>
<span class="linenos">182</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">183</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.00347222222222222</span><span class="w"></span>
<span class="linenos">184</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">185</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">186</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">643</span><span class="p">,</span><span class="w"></span>
<span class="linenos">187</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.18973561430793e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">188</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">189</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">101.973561430793</span><span class="p">,</span><span class="w"></span>
<span class="linenos">190</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">191</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.012441679627</span><span class="p">,</span><span class="w"></span>
<span class="linenos">192</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">193</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.0031104199066874</span><span class="w"></span>
<span class="linenos">194</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">195</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">196</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">591</span><span class="p">,</span><span class="w"></span>
<span class="linenos">197</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.1912013536379e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">198</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">199</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">101.944162436548</span><span class="p">,</span><span class="w"></span>
<span class="linenos">200</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">201</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.013536379019</span><span class="p">,</span><span class="w"></span>
<span class="linenos">202</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">203</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.00169204737732657</span><span class="w"></span>
<span class="linenos">204</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">205</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">206</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">673</span><span class="p">,</span><span class="w"></span>
<span class="linenos">207</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.19049034175334e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">208</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">209</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">101.973254086181</span><span class="p">,</span><span class="w"></span>
<span class="linenos">210</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">211</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.011887072808</span><span class="p">,</span><span class="w"></span>
<span class="linenos">212</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">213</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.00297176820208024</span><span class="w"></span>
<span class="linenos">214</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">215</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">216</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">649</span><span class="p">,</span><span class="w"></span>
<span class="linenos">217</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.19013867488444e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">218</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">219</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">101.850539291217</span><span class="p">,</span><span class="w"></span>
<span class="linenos">220</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">221</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.012326656394</span><span class="p">,</span><span class="w"></span>
<span class="linenos">222</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">223</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.00308166409861325</span><span class="w"></span>
<span class="linenos">224</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">225</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">226</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">606</span><span class="p">,</span><span class="w"></span>
<span class="linenos">227</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.18547854785479e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">228</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">229</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">101.83498349835</span><span class="p">,</span><span class="w"></span>
<span class="linenos">230</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">231</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.013201320132</span><span class="p">,</span><span class="w"></span>
<span class="linenos">232</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">233</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.0033003300330033</span><span class="w"></span>
<span class="linenos">234</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">235</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">236</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">650</span><span class="p">,</span><span class="w"></span>
<span class="linenos">237</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.18769230769231e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">238</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">239</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">101.898461538462</span><span class="p">,</span><span class="w"></span>
<span class="linenos">240</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">241</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.012307692308</span><span class="p">,</span><span class="w"></span>
<span class="linenos">242</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">243</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.00307692307692308</span><span class="w"></span>
<span class="linenos">244</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">245</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">246</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">615</span><span class="p">,</span><span class="w"></span>
<span class="linenos">247</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.18520325203252e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">248</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">249</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">101.858536585366</span><span class="p">,</span><span class="w"></span>
<span class="linenos">250</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">251</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.013008130081</span><span class="p">,</span><span class="w"></span>
<span class="linenos">252</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">253</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.0032520325203252</span><span class="w"></span>
<span class="linenos">254</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">255</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">256</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">579</span><span class="p">,</span><span class="w"></span>
<span class="linenos">257</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.18618307426598e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">258</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">259</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">101.989637305699</span><span class="p">,</span><span class="w"></span>
<span class="linenos">260</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">261</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.013816925734</span><span class="p">,</span><span class="w"></span>
<span class="linenos">262</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">263</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.00345423143350604</span><span class="w"></span>
<span class="linenos">264</span><span class="w">                </span><span class="p">},</span><span class="w"></span>
<span class="linenos">265</span><span class="w">                </span><span class="p">{</span><span class="w"></span>
<span class="linenos">266</span><span class="w">                    </span><span class="nt">&quot;iterations&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">657</span><span class="p">,</span><span class="w"></span>
<span class="linenos">267</span><span class="w">                    </span><span class="nt">&quot;elapsed&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">3.19558599695586e-08</span><span class="p">,</span><span class="w"></span>
<span class="linenos">268</span><span class="w">                    </span><span class="nt">&quot;pagefaults&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">269</span><span class="w">                    </span><span class="nt">&quot;cpucycles&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">102.229832572298</span><span class="p">,</span><span class="w"></span>
<span class="linenos">270</span><span class="w">                    </span><span class="nt">&quot;contextswitches&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span><span class="w"></span>
<span class="linenos">271</span><span class="w">                    </span><span class="nt">&quot;instructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">170.012176560122</span><span class="p">,</span><span class="w"></span>
<span class="linenos">272</span><span class="w">                    </span><span class="nt">&quot;branchinstructions&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">28</span><span class="p">,</span><span class="w"></span>
<span class="linenos">273</span><span class="w">                    </span><span class="nt">&quot;branchmisses&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.0030441400304414</span><span class="w"></span>
<span class="linenos">274</span><span class="w">                </span><span class="p">}</span><span class="w"></span>
<span class="linenos">275</span><span class="w">            </span><span class="p">]</span><span class="w"></span>
<span class="linenos">276</span><span class="w">        </span><span class="p">}</span><span class="w"></span>
<span class="linenos">277</span><span class="w">    </span><span class="p">]</span><span class="w"></span>
<span class="linenos">278</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</section>
<section id="pyperf-python-pyperf-module-output">
<span id="tutorial-template-pyperf"></span><h2>pyperf - Python pyperf module Output<a class="headerlink" href="#pyperf-python-pyperf-module-output" title="Permalink to this heading"></a></h2>
<p><a class="reference external" href="https://pyperf.readthedocs.io/en/latest/">Pyperf</a> is a powerful tool for benchmarking and system tuning, and it can also analyze
benchmark results. This template allows generation of output so it can be used for further analysis with pyperf.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Pyperf supports only a single benchmark result per generated output, so it is best to create a new
<code class="docutils literal notranslate"><span class="pre">Bench</span></code> object for each benchmark.</p>
</div>
<p>The template looks like this. Note that it directly makes use of <code class="docutils literal notranslate"><span class="pre">{{#measurement}}</span></code>, which is only possible when there is a single result in the benchmark.</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span>{
<span class="linenos"> 2</span>    &quot;benchmarks&quot;: [
<span class="linenos"> 3</span>        {
<span class="linenos"> 4</span>            &quot;runs&quot;: [
<span class="linenos"> 5</span>                {
<span class="linenos"> 6</span>                    &quot;values&quot;: [
<span class="linenos"> 7</span>{{#measurement}}                        {{elapsed}}{{^-last}},
<span class="linenos"> 8</span>{{/last}}{{/measurement}}
<span class="linenos"> 9</span>                    ]
<span class="linenos">10</span>                }
<span class="linenos">11</span>            ]
<span class="linenos">12</span>        }
<span class="linenos">13</span>    ],
<span class="linenos">14</span>    &quot;metadata&quot;: {
<span class="linenos">15</span>        &quot;loops&quot;: {{sum(iterations)}},
<span class="linenos">16</span>        &quot;inner_loops&quot;: {{batch}},
<span class="linenos">17</span>        &quot;name&quot;: &quot;{{title}}&quot;,
<span class="linenos">18</span>        &quot;unit&quot;: &quot;second&quot;
<span class="linenos">19</span>    },
<span class="linenos">20</span>    &quot;version&quot;: &quot;1.0&quot;
<span class="linenos">21</span>}
</pre></div>
</div>
<p>Here is an example that generates pyperf compatible output for a benchmark that shuffles a vector:</p>
<div class="literal-block-wrapper docutils container" id="id14">
<div class="code-block-caption"><span class="caption-text">example_pyperf.cpp</span><a class="headerlink" href="#id14" title="Permalink to this code"></a></div>
<div class="highlight-cpp notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;nanobench.h&gt;</span><span class="cp"></span>
<span class="linenos"> 2</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;thirdparty/doctest/doctest.h&gt;</span><span class="cp"></span>
<span class="linenos"> 3</span>
<span class="linenos"> 4</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;algorithm&gt;</span><span class="cp"></span>
<span class="linenos"> 5</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;fstream&gt;</span><span class="cp"></span>
<span class="linenos"> 6</span><span class="cp">#include</span><span class="w"> </span><span class="cpf">&lt;random&gt;</span><span class="cp"></span>
<span class="linenos"> 7</span>
<span class="linenos"> 8</span><span class="n">TEST_CASE</span><span class="p">(</span><span class="s">&quot;shuffle_pyperf&quot;</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos"> 9</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o">&lt;</span><span class="kt">uint64_t</span><span class="o">&gt;</span><span class="w"> </span><span class="n">data</span><span class="p">(</span><span class="mi">500</span><span class="p">,</span><span class="w"> </span><span class="mi">0</span><span class="p">);</span><span class="w"> </span><span class="c1">// input data for shuffling</span>
<span class="linenos">10</span>
<span class="linenos">11</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">default_random_engine</span><span class="w"> </span><span class="nf">defaultRng</span><span class="p">(</span><span class="mi">123</span><span class="p">);</span><span class="w"></span>
<span class="linenos">12</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">ofstream</span><span class="w"> </span><span class="n">fout1</span><span class="p">(</span><span class="s">&quot;pyperf_shuffle_std.json&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">13</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">()</span><span class="w"></span>
<span class="linenos">14</span><span class="w">        </span><span class="p">.</span><span class="n">epochs</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span><span class="w"></span>
<span class="linenos">15</span><span class="w">        </span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;std::shuffle with std::default_random_engine&quot;</span><span class="p">,</span><span class="w"></span>
<span class="linenos">16</span><span class="w">             </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]()</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">17</span><span class="w">                 </span><span class="n">std</span><span class="o">::</span><span class="n">shuffle</span><span class="p">(</span><span class="n">data</span><span class="p">.</span><span class="n">begin</span><span class="p">(),</span><span class="w"> </span><span class="n">data</span><span class="p">.</span><span class="n">end</span><span class="p">(),</span><span class="w"> </span><span class="n">defaultRng</span><span class="p">);</span><span class="w"></span>
<span class="linenos">18</span><span class="w">             </span><span class="p">})</span><span class="w"></span>
<span class="linenos">19</span><span class="w">        </span><span class="p">.</span><span class="n">render</span><span class="p">(</span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">templates</span><span class="o">::</span><span class="n">pyperf</span><span class="p">(),</span><span class="w"> </span><span class="n">fout1</span><span class="p">);</span><span class="w"></span>
<span class="linenos">20</span>
<span class="linenos">21</span><span class="w">    </span><span class="n">std</span><span class="o">::</span><span class="n">ofstream</span><span class="w"> </span><span class="n">fout2</span><span class="p">(</span><span class="s">&quot;pyperf_shuffle_nanobench.json&quot;</span><span class="p">);</span><span class="w"></span>
<span class="linenos">22</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Rng</span><span class="w"> </span><span class="nf">rng</span><span class="p">(</span><span class="mi">123</span><span class="p">);</span><span class="w"></span>
<span class="linenos">23</span><span class="w">    </span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">Bench</span><span class="p">()</span><span class="w"></span>
<span class="linenos">24</span><span class="w">        </span><span class="p">.</span><span class="n">epochs</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span><span class="w"></span>
<span class="linenos">25</span><span class="w">        </span><span class="p">.</span><span class="n">run</span><span class="p">(</span><span class="s">&quot;ankerl::nanobench::Rng::shuffle&quot;</span><span class="p">,</span><span class="w"></span>
<span class="linenos">26</span><span class="w">             </span><span class="p">[</span><span class="o">&amp;</span><span class="p">]()</span><span class="w"> </span><span class="p">{</span><span class="w"></span>
<span class="linenos">27</span><span class="w">                 </span><span class="n">rng</span><span class="p">.</span><span class="n">shuffle</span><span class="p">(</span><span class="n">data</span><span class="p">);</span><span class="w"></span>
<span class="linenos">28</span><span class="w">             </span><span class="p">})</span><span class="w"></span>
<span class="linenos">29</span><span class="w">        </span><span class="p">.</span><span class="n">render</span><span class="p">(</span><span class="n">ankerl</span><span class="o">::</span><span class="n">nanobench</span><span class="o">::</span><span class="n">templates</span><span class="o">::</span><span class="n">pyperf</span><span class="p">(),</span><span class="w"> </span><span class="n">fout2</span><span class="p">);</span><span class="w"></span>
<span class="linenos">30</span><span class="p">}</span><span class="w"></span>
</pre></div>
</div>
</div>
<p>This benchmark run creates the two files <code class="docutils literal notranslate"><span class="pre">pyperf_shuffle_std.json</span></code> and <code class="docutils literal notranslate"><span class="pre">pyperf_shuffle_nanobench.json</span></code>.
Here are some of the analysis you can do:</p>
<section id="show-benchmark-statistics">
<h3>Show Benchmark Statistics<a class="headerlink" href="#show-benchmark-statistics" title="Permalink to this heading"></a></h3>
<p>Output from <code class="docutils literal notranslate"><span class="pre">python3</span> <span class="pre">-m</span> <span class="pre">pyperf</span> <span class="pre">stats</span> <span class="pre">pyperf_shuffle_std.json</span></code>:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>Total duration: 364 ms
Raw value minimum: 3.57 ms
Raw value maximum: 4.21 ms

Number of calibration run: 0
Number of run with values: 1
Total number of run: 1

Number of warmup per run: 0
Number of value per run: 100
Loop iterations per value: 100
Total number of values: 100

Minimum:         35.7 us
Median +- MAD:   36.2 us +- 0.2 us
Mean +- std dev: 36.4 us +- 0.9 us
Maximum:         42.1 us

  0th percentile: 35.7 us (-2% of the mean) -- minimum
  5th percentile: 35.8 us (-2% of the mean)
 25th percentile: 36.1 us (-1% of the mean) -- Q1
 50th percentile: 36.2 us (-0% of the mean) -- median
 75th percentile: 36.4 us (+0% of the mean) -- Q3
 95th percentile: 36.7 us (+1% of the mean)
100th percentile: 42.1 us (+16% of the mean) -- maximum

Number of outlier (out of 35.6 us..36.9 us): 4
</pre></div>
</div>
</section>
<section id="show-a-histogram">
<h3>Show a Histogram<a class="headerlink" href="#show-a-histogram" title="Permalink to this heading"></a></h3>
<p>It’s often interesting to see a histogram, especially to visually find out if there are outliers involved.
Run <code class="docutils literal notranslate"><span class="pre">python3</span> <span class="pre">-m</span> <span class="pre">pyperf</span> <span class="pre">hist</span> <span class="pre">pyperf_shuffle_std.json</span></code> produces this output</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>35.7 us: 21 ######################################
36.0 us: 33 ############################################################
36.3 us: 37 ###################################################################
36.6 us:  5 #########
36.9 us:  0 |
37.2 us:  1 ##
37.5 us:  0 |
37.8 us:  0 |
38.1 us:  0 |
38.4 us:  0 |
38.7 us:  0 |
39.0 us:  0 |
39.3 us:  0 |
39.6 us:  1 ##
39.9 us:  0 |
40.2 us:  0 |
40.5 us:  1 ##
40.8 us:  0 |
41.1 us:  0 |
41.5 us:  0 |
41.8 us:  0 |
42.1 us:  1 ##
</pre></div>
</div>
</section>
<section id="compare-results">
<h3>Compare Results<a class="headerlink" href="#compare-results" title="Permalink to this heading"></a></h3>
<p>We have generated two results in the above examples, and we can compare them easily with <code class="docutils literal notranslate"><span class="pre">python3</span> <span class="pre">-m</span> <span class="pre">pyperf</span> <span class="pre">compare_to</span> <span class="pre">a.json</span> <span class="pre">b.json</span></code>:</p>
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>+-----------+--------------------+------------------------------+
| Benchmark | pyperf_shuffle_std | pyperf_shuffle_nanobench     |
+===========+====================+==============================+
| benchmark | 36.4 us            | 11.2 us: 3.24x faster (-69%) |
+-----------+--------------------+------------------------------+
</pre></div>
</div>
<p>For more information of pyperfs analysis capability, please see <a class="reference external" href="https://pyperf.readthedocs.io/en/latest/analyze.html">pyperf - Analyze benchmark results</a>.</p>
</section>
</section>
</section>


           </div>
          </div>
          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
        <a href="index.html" class="btn btn-neutral float-left" title="nanobench" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
        <a href="comparison.html" class="btn btn-neutral float-right" title="Test Set" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
    </div>

  <hr/>

  <div role="contentinfo">
    <p>&#169; Copyright 2019-2022 Martin Leitner-Ankerl &lt;martin.ankerl@gmail.com&gt;.</p>
  </div>

  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
    provided by <a href="https://readthedocs.org">Read the Docs</a>.
   

</footer>
        </div>
      </div>
    </section>
  </div>
  <script>
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
    <!-- Theme Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-36863101-2"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());

      gtag('config', 'UA-36863101-2', {
          'anonymize_ip': false,
      });
    </script> 

</body>
</html>